diff --git a/examples/c/tmq.c b/examples/c/tmq.c index eb41ad039a..8a2112fbcc 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -189,27 +189,46 @@ void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { tmq_t* build_consumer() { tmq_conf_res_t code; + tmq_t* tmq = NULL; + tmq_conf_t* conf = tmq_conf_new(); code = tmq_conf_set(conf, "enable.auto.commit", "true"); - if (TMQ_CONF_OK != code) return NULL; + if (TMQ_CONF_OK != code) { + goto _end; + } code = tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); - if (TMQ_CONF_OK != code) return NULL; + if (TMQ_CONF_OK != code) { + goto _end; + } code = tmq_conf_set(conf, "group.id", "cgrpName"); - if (TMQ_CONF_OK != code) return NULL; + if (TMQ_CONF_OK != code) { + goto _end; + } code = tmq_conf_set(conf, "client.id", "user defined name"); - if (TMQ_CONF_OK != code) return NULL; + if (TMQ_CONF_OK != code) { + goto _end; + } code = tmq_conf_set(conf, "td.connect.user", "root"); - if (TMQ_CONF_OK != code) return NULL; + if (TMQ_CONF_OK != code) { + goto _end; + } code = tmq_conf_set(conf, "td.connect.pass", "taosdata"); - if (TMQ_CONF_OK != code) return NULL; + if (TMQ_CONF_OK != code) { + goto _end; + } code = tmq_conf_set(conf, "auto.offset.reset", "earliest"); - if (TMQ_CONF_OK != code) return NULL; + if (TMQ_CONF_OK != code) { + goto _end; + } code = tmq_conf_set(conf, "experimental.snapshot.enable", "false"); - if (TMQ_CONF_OK != code) return NULL; + if (TMQ_CONF_OK != code) { + goto _end; + } tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); + tmq = tmq_consumer_new(conf, NULL, 0); - tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); + _end: tmq_conf_destroy(conf); return tmq; } diff --git a/include/common/tcommon.h b/include/common/tcommon.h index f4e13509c2..0ea78a00ff 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -205,7 +205,8 @@ typedef struct SDataBlockInfo { STimeWindow calWin; // used for stream, do not serialize TSKEY watermark; // used for stream - char parTbName[TSDB_TABLE_NAME_LEN]; // used for stream partition + char parTbName[TSDB_TABLE_NAME_LEN]; // used for stream partition + STag* pTag; // used for stream partition } SDataBlockInfo; typedef struct SSDataBlock { @@ -341,7 +342,7 @@ typedef struct SExprInfo { typedef struct { const char* key; - size_t keyLen; + size_t keyLen; uint8_t type; union { const char* value; @@ -385,9 +386,9 @@ typedef struct STUidTagInfo { #define TABLE_NAME_COLUMN_INDEX 6 // stream create table block column -#define UD_TABLE_NAME_COLUMN_INDEX 0 -#define UD_GROUPID_COLUMN_INDEX 1 -#define UD_TAG_COLUMN_INDEX 2 +#define UD_TABLE_NAME_COLUMN_INDEX 0 +#define UD_GROUPID_COLUMN_INDEX 1 +#define UD_TAG_COLUMN_INDEX 2 #ifdef __cplusplus } diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 20ffb48ab0..84d696e518 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -58,7 +58,6 @@ typedef struct SBlockOrderInfo { #define BitmapLen(_n) (((_n) + ((1 << NBIT) - 1)) >> NBIT) #define colDataGetVarData(p1_, r_) ((p1_)->pData + (p1_)->varmeta.offset[(r_)]) - #define colDataGetNumData(p1_, r_) ((p1_)->pData + ((r_) * (p1_)->info.bytes)) // SColumnInfoData, rowNumber #define colDataGetData(p1_, r_) \ @@ -83,33 +82,6 @@ static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, } } -static FORCE_INLINE bool colDataIsNNull_s(const SColumnInfoData* pColumnInfoData, int32_t startIndex, - uint32_t nRows) { - if (!pColumnInfoData->hasNull) { - return false; - } - - if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { - for (int32_t i = startIndex; i < nRows; ++i) { - if (!colDataIsNull_var(pColumnInfoData, i)) { - return false; - } - } - } else { - if (pColumnInfoData->nullbitmap == NULL) { - return false; - } - - for (int32_t i = startIndex; i < nRows; ++i) { - if (!colDataIsNull_f(pColumnInfoData->nullbitmap, i)) { - return false; - } - } - } - - return true; -} - static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, uint32_t totalRows, uint32_t row, SColumnDataAgg* pColAgg) { if (!pColumnInfoData->hasNull) { @@ -137,18 +109,18 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u } } -static FORCE_INLINE void colDataAppendNULL(SColumnInfoData* pColumnInfoData, uint32_t currentRow) { +static FORCE_INLINE void colDataSetNULL(SColumnInfoData* pColumnInfoData, uint32_t rowIndex) { // There is a placehold for each NULL value of binary or nchar type. if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { - colDataSetNull_var(pColumnInfoData, currentRow); // it is a null value of VAR type. + colDataSetNull_var(pColumnInfoData, rowIndex); // it is a null value of VAR type. } else { - colDataSetNull_f_s(pColumnInfoData, currentRow); + colDataSetNull_f_s(pColumnInfoData, rowIndex); } pColumnInfoData->hasNull = true; } -static FORCE_INLINE void colDataAppendNNULL(SColumnInfoData* pColumnInfoData, uint32_t start, size_t nRows) { +static FORCE_INLINE void colDataSetNNULL(SColumnInfoData* pColumnInfoData, uint32_t start, size_t nRows) { if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { for (int32_t i = start; i < start + nRows; ++i) { colDataSetNull_var(pColumnInfoData, i); // it is a null value of VAR type. @@ -163,50 +135,50 @@ static FORCE_INLINE void colDataAppendNNULL(SColumnInfoData* pColumnInfoData, ui pColumnInfoData->hasNull = true; } -static FORCE_INLINE void colDataAppendInt8(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int8_t* v) { +static FORCE_INLINE void colDataSetInt8(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, int8_t* v) { ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_TINYINT || pColumnInfoData->info.type == TSDB_DATA_TYPE_UTINYINT || pColumnInfoData->info.type == TSDB_DATA_TYPE_BOOL); - char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow; + char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * rowIndex; *(int8_t*)p = *(int8_t*)v; } -static FORCE_INLINE void colDataAppendInt16(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int16_t* v) { +static FORCE_INLINE void colDataSetInt16(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, int16_t* v) { ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_SMALLINT || pColumnInfoData->info.type == TSDB_DATA_TYPE_USMALLINT); - char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow; + char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * rowIndex; *(int16_t*)p = *(int16_t*)v; } -static FORCE_INLINE void colDataAppendInt32(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int32_t* v) { +static FORCE_INLINE void colDataSetInt32(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, int32_t* v) { ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_INT || pColumnInfoData->info.type == TSDB_DATA_TYPE_UINT); - char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow; + char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * rowIndex; *(int32_t*)p = *(int32_t*)v; } -static FORCE_INLINE void colDataAppendInt64(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int64_t* v) { +static FORCE_INLINE void colDataSetInt64(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, int64_t* v) { int32_t type = pColumnInfoData->info.type; ASSERT(type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_UBIGINT || type == TSDB_DATA_TYPE_TIMESTAMP); - char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow; + char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * rowIndex; *(int64_t*)p = *(int64_t*)v; } -static FORCE_INLINE void colDataAppendFloat(SColumnInfoData* pColumnInfoData, uint32_t currentRow, float* v) { +static FORCE_INLINE void colDataSetFloat(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, float* v) { ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_FLOAT); - char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow; + char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * rowIndex; *(float*)p = *(float*)v; } -static FORCE_INLINE void colDataAppendDouble(SColumnInfoData* pColumnInfoData, uint32_t currentRow, double* v) { +static FORCE_INLINE void colDataSetDouble(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, double* v) { ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_DOUBLE); - char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow; + char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * rowIndex; *(double*)p = *(double*)v; } int32_t getJsonValueLen(const char* data); -int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull); -int32_t colDataAppendNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, - uint32_t numOfRows); +int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull); +int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull); +int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, uint32_t numOfRows); int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int32_t* capacity, const SColumnInfoData* pSource, int32_t numOfRow2); int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows, @@ -245,7 +217,7 @@ void blockDataEmpty(SSDataBlock* pDataBlock); size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize); -int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n); +int32_t blockDataTrimFirstRows(SSDataBlock* pBlock, size_t n); int32_t blockDataKeepFirstNRows(SSDataBlock* pBlock, size_t n); int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src); @@ -280,13 +252,6 @@ static FORCE_INLINE int32_t blockGetEncodeSize(const SSDataBlock* pBlock) { return blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock)) + blockDataGetSize(pBlock); } -static FORCE_INLINE int32_t blockCompressColData(SColumnInfoData* pColRes, int32_t numOfRows, char* data, - int8_t compressed) { - int32_t colSize = colDataGetLength(pColRes, numOfRows); - return (*(tDataTypes[pColRes->info.type].compFunc))(pColRes->pData, colSize, numOfRows, data, - colSize + COMP_OVERFLOW_BYTES, compressed, NULL, 0); -} - #ifdef __cplusplus } #endif diff --git a/include/common/tmisce.h b/include/common/tmisce.h index b9f5cf5b91..bc6558900c 100644 --- a/include/common/tmisce.h +++ b/include/common/tmisce.h @@ -27,13 +27,14 @@ typedef struct SCorEpSet { SEpSet epSet; } SCorEpSet; +#define GET_ACTIVE_EP(_eps) (&((_eps)->eps[(_eps)->inUse])) int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp); void addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port); -bool isEpsetEqual(const SEpSet* s1, const SEpSet* s2); - -void updateEpSet_s(SCorEpSet* pEpSet, SEpSet* pNewEpSet); -SEpSet getEpSet_s(SCorEpSet* pEpSet); +bool isEpsetEqual(const SEpSet* s1, const SEpSet* s2); +void epsetAssign(SEpSet* dst, const SEpSet* pSrc); +void updateEpSet_s(SCorEpSet* pEpSet, SEpSet* pNewEpSet); +SEpSet getEpSet_s(SCorEpSet* pEpSet); #ifdef __cplusplus } diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 77b9d2d681..98fcc2effa 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1904,10 +1904,10 @@ typedef struct { } SMqConsumerLostMsg, SMqConsumerRecoverMsg, SMqConsumerClearMsg; typedef struct { - int64_t consumerId; - char cgroup[TSDB_CGROUP_LEN]; - char clientId[256]; - SArray* topicNames; // SArray + int64_t consumerId; + char cgroup[TSDB_CGROUP_LEN]; + char clientId[256]; + SArray* topicNames; // SArray } SCMSubscribeReq; static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubscribeReq* pReq) { diff --git a/include/util/tarray.h b/include/util/tarray.h index 5b36bc4d0e..a8510e4bc8 100644 --- a/include/util/tarray.h +++ b/include/util/tarray.h @@ -43,7 +43,7 @@ typedef struct SArray { * @return */ SArray* taosArrayInit(size_t size, size_t elemSize); -SArray* taosArrayInit_s(size_t size, size_t elemSize, size_t initialSize); +SArray* taosArrayInit_s(size_t elemSize, size_t initialSize); /** * diff --git a/include/util/tworker.h b/include/util/tworker.h index 0636f16dbb..8508adf052 100644 --- a/include/util/tworker.h +++ b/include/util/tworker.h @@ -26,12 +26,12 @@ extern "C" { typedef struct SQWorkerPool SQWorkerPool; typedef struct SWWorkerPool SWWorkerPool; -typedef struct SQWorker { +typedef struct SQueueWorker { int32_t id; // worker id int64_t pid; // thread pid TdThread thread; // thread id void *pool; -} SQWorker; +} SQueueWorker; typedef struct SQWorkerPool { int32_t max; // max number of workers @@ -39,7 +39,7 @@ typedef struct SQWorkerPool { int32_t num; // current number of workers STaosQset *qset; const char *name; - SQWorker *workers; + SQueueWorker *workers; TdThreadMutex mutex; } SQWorkerPool; diff --git a/include/util/xxhash.h b/include/util/xxhash.h deleted file mode 100644 index d6bad94335..0000000000 --- a/include/util/xxhash.h +++ /dev/null @@ -1,328 +0,0 @@ -/* - xxHash - Extremely Fast Hash algorithm - Header File - Copyright (C) 2012-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - xxHash source repository : https://github.com/Cyan4973/xxHash -*/ - -/* Notice extracted from xxHash homepage : - -xxHash is an extremely fast Hash algorithm, running at RAM speed limits. -It also successfully passes all tests from the SMHasher suite. - -Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) - -Name Speed Q.Score Author -xxHash 5.4 GB/s 10 -CrapWow 3.2 GB/s 2 Andrew -MumurHash 3a 2.7 GB/s 10 Austin Appleby -SpookyHash 2.0 GB/s 10 Bob Jenkins -SBox 1.4 GB/s 9 Bret Mulvey -Lookup3 1.2 GB/s 9 Bob Jenkins -SuperFastHash 1.2 GB/s 1 Paul Hsieh -CityHash64 1.05 GB/s 10 Pike & Alakuijala -FNV 0.55 GB/s 5 Fowler, Noll, Vo -CRC32 0.43 GB/s 9 -MD5-32 0.33 GB/s 10 Ronald L. Rivest -SHA1-32 0.28 GB/s 10 - -Q.Score is a measure of quality of the hash function. -It depends on successfully passing SMHasher test set. -10 is a perfect score. - -A 64-bit version, named XXH64, is available since r35. -It offers much better speed, but for 64-bit applications only. -Name Speed on 64 bits Speed on 32 bits -XXH64 13.8 GB/s 1.9 GB/s -XXH32 6.8 GB/s 6.0 GB/s -*/ - -#ifndef XXHASH_H_5627135585666179 -#define XXHASH_H_5627135585666179 1 - -#if defined (__cplusplus) -extern "C" { -#endif - - -/* **************************** -* Definitions -******************************/ -#include /* size_t */ -typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; - - -/* **************************** - * API modifier - ******************************/ -/** XXH_INLINE_ALL (and XXH_PRIVATE_API) - * This is useful to include xxhash functions in `static` mode - * in order to inline them, and remove their symbol from the public list. - * Inlining can offer dramatic performance improvement on small keys. - * Methodology : - * #define XXH_INLINE_ALL - * #include "xxhash.h" - * `xxhash.c` is automatically included. - * It's not useful to compile and link it as a separate module. - */ -#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) -# ifndef XXH_STATIC_LINKING_ONLY -# define XXH_STATIC_LINKING_ONLY -# endif -# if defined(__GNUC__) -# define XXH_PUBLIC_API static __inline __attribute__((unused)) -# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define XXH_PUBLIC_API static inline -# elif defined(_MSC_VER) -# define XXH_PUBLIC_API static __inline -# else - /* this version may generate warnings for unused static functions */ -# define XXH_PUBLIC_API static -# endif -#else -# define XXH_PUBLIC_API /* do nothing */ -#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */ - -/*! XXH_NAMESPACE, aka Namespace Emulation : - * - * If you want to include _and expose_ xxHash functions from within your own library, - * but also want to avoid symbol collisions with other libraries which may also include xxHash, - * - * you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library - * with the value of XXH_NAMESPACE (therefore, avoid NULL and numeric values). - * - * Note that no change is required within the calling program as long as it includes `xxhash.h` : - * regular symbol name will be automatically translated by this header. - */ -#ifdef XXH_NAMESPACE -# define XXH_CAT(A,B) A##B -# define XXH_NAME2(A,B) XXH_CAT(A,B) -# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) -# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) -# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) -# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) -# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) -# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) -# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) -# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) -# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) -# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) -# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) -# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) -# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) -# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) -# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) -# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) -# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) -# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) -# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) -#endif - - -/* ************************************* -* Version -***************************************/ -#define XXH_VERSION_MAJOR 0 -#define XXH_VERSION_MINOR 6 -#define XXH_VERSION_RELEASE 5 -#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) -XXH_PUBLIC_API unsigned XXH_versionNumber (void); - - -/*-********************************************************************** -* 32-bit hash -************************************************************************/ -typedef unsigned int XXH32_hash_t; - -/*! XXH32() : - Calculate the 32-bit hash of sequence "length" bytes stored at memory address "input". - The memory between input & input+length must be valid (allocated and read-accessible). - "seed" can be used to alter the result predictably. - Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s */ -XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed); - -/*====== Streaming ======*/ -typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */ -XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); -XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); -XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state); - -XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned int seed); -XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); -XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); - -/* - * Streaming functions generate the xxHash of an input provided in multiple segments. - * Note that, for small input, they are slower than single-call functions, due to state management. - * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized. - * - * XXH state must first be allocated, using XXH*_createState() . - * - * Start a new hash by initializing state with a seed, using XXH*_reset(). - * - * Then, feed the hash state by calling XXH*_update() as many times as necessary. - * The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. - * - * Finally, a hash value can be produced anytime, by using XXH*_digest(). - * This function returns the nn-bits hash as an int or long long. - * - * It's still possible to continue inserting input into the hash state after a digest, - * and generate some new hashes later on, by calling again XXH*_digest(). - * - * When done, free XXH state space if it was allocated dynamically. - */ - -/*====== Canonical representation ======*/ - -typedef struct { unsigned char digest[4]; } XXH32_canonical_t; -XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); -XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); - -/* Default result type for XXH functions are primitive unsigned 32 and 64 bits. - * The canonical representation uses human-readable write convention, aka big-endian (large digits first). - * These functions allow transformation of hash result into and from its canonical format. - * This way, hash values can be written into a file / memory, and remain comparable on different systems and programs. - */ - - -#ifndef XXH_NO_LONG_LONG -/*-********************************************************************** -* 64-bit hash -************************************************************************/ -typedef unsigned long long XXH64_hash_t; - -/*! XXH64() : - Calculate the 64-bit hash of sequence of length "len" stored at memory address "input". - "seed" can be used to alter the result predictably. - This function runs faster on 64-bit systems, but slower on 32-bit systems (see benchmark). -*/ -XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed); - -/*====== Streaming ======*/ -typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ -XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void); -XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); -XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dst_state, const XXH64_state_t* src_state); - -XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed); -XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); -XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr); - -/*====== Canonical representation ======*/ -typedef struct { unsigned char digest[8]; } XXH64_canonical_t; -XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash); -XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); -#endif /* XXH_NO_LONG_LONG */ - - - -#ifdef XXH_STATIC_LINKING_ONLY - -/* ================================================================================================ - This section contains declarations which are not guaranteed to remain stable. - They may change in future versions, becoming incompatible with a different version of the library. - These declarations should only be used with static linking. - Never use them in association with dynamic linking ! -=================================================================================================== */ - -/* These definitions are only present to allow - * static allocation of XXH state, on stack or in a struct for example. - * Never **ever** use members directly. */ - -#if !defined (__VMS) \ - && (defined (__cplusplus) \ - || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) -# include - -struct XXH32_state_s { - uint32_t total_len_32; - uint32_t large_len; - uint32_t v1; - uint32_t v2; - uint32_t v3; - uint32_t v4; - uint32_t mem32[4]; - uint32_t memsize; - uint32_t reserved; /* never read nor write, might be removed in a future version */ -}; /* typedef'd to XXH32_state_t */ - -struct XXH64_state_s { - uint64_t total_len; - uint64_t v1; - uint64_t v2; - uint64_t v3; - uint64_t v4; - uint64_t mem64[4]; - uint32_t memsize; - uint32_t reserved[2]; /* never read nor write, might be removed in a future version */ -}; /* typedef'd to XXH64_state_t */ - -# else - -struct XXH32_state_s { - unsigned total_len_32; - unsigned large_len; - unsigned v1; - unsigned v2; - unsigned v3; - unsigned v4; - unsigned mem32[4]; - unsigned memsize; - unsigned reserved; /* never read nor write, might be removed in a future version */ -}; /* typedef'd to XXH32_state_t */ - -# ifndef XXH_NO_LONG_LONG /* remove 64-bit support */ -struct XXH64_state_s { - unsigned long long total_len; - unsigned long long v1; - unsigned long long v2; - unsigned long long v3; - unsigned long long v4; - unsigned long long mem64[4]; - unsigned memsize; - unsigned reserved[2]; /* never read nor write, might be removed in a future version */ -}; /* typedef'd to XXH64_state_t */ -# endif - -# endif - - -#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) -# include "xxhash.c" /* include xxhash function bodies as `static`, for inlining */ -#endif - -#endif /* XXH_STATIC_LINKING_ONLY */ - - -#if defined (__cplusplus) -} -#endif - -#endif /* XXHASH_H_5627135585666179 */ diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 3cb8a2e1bd..92d15f4887 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -347,7 +347,7 @@ int32_t hbBuildQueryDesc(SQueryHbReqBasic *hbBasic, STscObj *pObj) { continue; } - if (pRequest->killed) { + if (pRequest->killed || 0 == pRequest->body.queryJob) { releaseRequest(*rid); pIter = taosHashIterate(pObj->pRequests, pIter); continue; diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 07624efe04..409c19f65e 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -418,12 +418,12 @@ static int32_t buildShowVariablesBlock(SArray* pVars, SSDataBlock** block) { char name[TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(name, pInfo->name, TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE); SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, c++); - colDataAppend(pColInfo, i, name, false); + colDataSetVal(pColInfo, i, name, false); char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(value, pInfo->value, TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE); pColInfo = taosArrayGet(pBlock->pDataBlock, c++); - colDataAppend(pColInfo, i, value, false); + colDataSetVal(pColInfo, i, value, false); } pBlock->info.rows = numOfCfg; diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index f2c212928e..a2f752edb9 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -544,7 +544,7 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin } } else { if (currElement->colArray == NULL) { - currElement->colArray = taosArrayInit_s(16, sizeof(SSmlKv), 1); + currElement->colArray = taosArrayInit_s(sizeof(SSmlKv), 1); } taosArrayPush(currElement->colArray, &kv); // reserve for timestamp } diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 540cec1de3..836bcd7473 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -58,15 +58,14 @@ struct tmq_list_t { }; struct tmq_conf_t { - char clientId[256]; - char groupId[TSDB_CGROUP_LEN]; - int8_t autoCommit; - int8_t resetOffset; - int8_t withTbName; - int8_t snapEnable; - int32_t snapBatchSize; - bool hbBgEnable; - + char clientId[256]; + char groupId[TSDB_CGROUP_LEN]; + int8_t autoCommit; + int8_t resetOffset; + int8_t withTbName; + int8_t snapEnable; + int32_t snapBatchSize; + bool hbBgEnable; uint16_t port; int32_t autoCommitInterval; char* ip; @@ -213,6 +212,7 @@ typedef struct { typedef struct { SMqCommitCbParamSet* params; STqOffset* pOffset; + SMqClientVg* pMqVg; /*char topicName[TSDB_TOPIC_FNAME_LEN];*/ /*int32_t vgId;*/ } SMqCommitCbParam; @@ -422,10 +422,6 @@ int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet) { static void tmqCommitRspCountDown(SMqCommitCbParamSet* pParamSet) { int32_t waitingRspNum = atomic_sub_fetch_32(&pParamSet->waitingRspNum, 1); - if (ASSERT(waitingRspNum >= 0)) { - tscError("tmqCommitRspCountDown error:%d", waitingRspNum); - return; - } if (waitingRspNum == 0) { tmqCommitDone(pParamSet); } @@ -443,6 +439,17 @@ int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) { } #endif + // there may be race condition. fix it + if (pBuf->pEpSet != NULL && pParam->pMqVg != NULL) { + SMqClientVg* pMqVg = pParam->pMqVg; + + SEp* pEp = GET_ACTIVE_EP(pBuf->pEpSet); + SEp* pOld = GET_ACTIVE_EP(&(pMqVg->epSet)); + uDebug("subKey:%s update the epset vgId:%d, ep:%s:%d, old ep:%s:%d", pParam->pOffset->subKey, pMqVg->vgId, + pEp->fqdn, pEp->port, pOld->fqdn, pOld->port); + pParam->pMqVg->epSet = *pBuf->pEpSet; + } + taosMemoryFree(pParam->pOffset); taosMemoryFree(pBuf->pData); taosMemoryFree(pBuf->pEpSet); @@ -451,7 +458,6 @@ int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) { * pOffset->version);*/ tmqCommitRspCountDown(pParamSet); - return 0; } @@ -461,6 +467,7 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } + pOffset->val = pVg->currentOffset; int32_t groupLen = strlen(tmq->groupId); @@ -474,11 +481,13 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT if (code < 0) { return -1; } + void* buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len); if (buf == NULL) { taosMemoryFree(pOffset); return -1; } + ((SMsgHead*)buf)->vgId = htonl(pVg->vgId); void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); @@ -495,8 +504,10 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT taosMemoryFree(buf); return -1; } + pParam->params = pParamSet; pParam->pOffset = pOffset; + pParam->pMqVg = pVg; // there may be an race condition // build send info SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); @@ -506,16 +517,18 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT taosMemoryFree(pParam); return -1; } + pMsgSendInfo->msgInfo = (SDataBuf){ .pData = buf, .len = sizeof(SMsgHead) + len, .handle = NULL, }; - tscDebug("consumer:0x%" PRIx64 " topic:%s on vgId:%d offset:%" PRId64, tmq->consumerId, pOffset->subKey, pVg->vgId, - pOffset->val.version); + SEp* pEp = &pVg->epSet.eps[pVg->epSet.inUse]; + tscDebug("consumer:0x%" PRIx64 " topic:%s on vgId:%d offset:%" PRId64 " prev:%" PRId64 ", ep:%s:%d", tmq->consumerId, + pOffset->subKey, pVg->vgId, pOffset->val.version, pVg->committedOffset.version, pEp->fqdn, pEp->port); - // TODO: put into cb + // TODO: put into cb, the commit offset should be move to the callback function pVg->committedOffset = pVg->currentOffset; pMsgSendInfo->requestId = generateRequestId(); @@ -639,15 +652,16 @@ static int32_t tmqCommitConsumerImpl(tmq_t* tmq, int8_t automatic, int8_t async, // init as 1 to prevent concurrency issue pParamSet->waitingRspNum = 1; - for (int32_t i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { + int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics); + tscDebug("consumer:0x%" PRIx64 " start to commit offset for %d topics", tmq->consumerId, numOfTopics); + + for (int32_t i = 0; i < numOfTopics; i++) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs); for (int32_t j = 0; j < numOfVgroups; j++) { SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); if (pVg->currentOffset.type > 0 && !tOffsetEqual(&pVg->currentOffset, &pVg->committedOffset)) { - tscDebug("consumer:0x%" PRIx64 " topic:%s vgId:%d, current %" PRId64 ", committed %" PRId64, tmq->consumerId, - pTopic->topicName, pVg->vgId, pVg->currentOffset.version, pVg->committedOffset.version); if (tmqSendCommitReq(tmq, pVg, pTopic, pParamSet) < 0) { continue; } @@ -981,7 +995,8 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->qall = taosAllocateQall(); pTmq->delayedTask = taosOpenQueue(); - if (pTmq->clientTopics == NULL || pTmq->mqueue == NULL || pTmq->qall == NULL || pTmq->delayedTask == NULL) { + if (pTmq->clientTopics == NULL || pTmq->mqueue == NULL || pTmq->qall == NULL || pTmq->delayedTask == NULL || + conf->groupId[0] == 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; tscError("consumer:0x%" PRIx64 " setup failed since %s, consumer group %s", pTmq->consumerId, terrstr(), pTmq->groupId); @@ -1060,7 +1075,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { SCMSubscribeReq req = {0}; int32_t code = 0; - tscDebug("consumer:0x%" PRIx64 " tmq subscribe start, numOfTopic %d", tmq->consumerId, sz); + tscDebug("consumer:0x%" PRIx64 " subscribe %d topics", tmq->consumerId, sz); req.consumerId = tmq->consumerId; tstrncpy(req.clientId, tmq->clientId, 256); @@ -1083,7 +1098,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { } tNameExtractFullName(&name, topicFName); - tscDebug("consumer:0x%" PRIx64 ", subscribe topic: %s", tmq->consumerId, topicFName); + tscDebug("consumer:0x%" PRIx64 " subscribe topic:%s", tmq->consumerId, topicFName); taosArrayPush(req.topicNames, &topicFName); } @@ -1394,7 +1409,7 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) { } atomic_store_32(&tmq->epoch, epoch); - tscDebug("consumer:0x%" PRIx64 ", update topic info completed", tmq->consumerId); + tscDebug("consumer:0x%" PRIx64 " update topic info completed", tmq->consumerId); return set; } @@ -1544,7 +1559,8 @@ int32_t tmqAskEp(tmq_t* tmq, bool async) { sendInfo->msgType = TDMT_MND_TMQ_ASK_EP; SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); - tscDebug("consumer:0x%" PRIx64 " ask ep from mnode, async:%d", tmq->consumerId, async); + tscDebug("consumer:0x%" PRIx64 " ask ep from mnode, async:%d, reqId:0x%" PRIx64, tmq->consumerId, async, + tmq->consumerId); int64_t transporterId = 0; asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); @@ -1751,28 +1767,28 @@ int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* pReset) } void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { + tscDebug("consumer:0x%" PRIx64 " start to handle the rsp", tmq->consumerId); + while (1) { SMqRspWrapper* rspWrapper = NULL; taosGetQitem(tmq->qall, (void**)&rspWrapper); + if (rspWrapper == NULL) { taosReadAllQitems(tmq->mqueue, tmq->qall); taosGetQitem(tmq->qall, (void**)&rspWrapper); if (rspWrapper == NULL) { - /*tscDebug("consumer %" PRId64 " mqueue empty", tmq->consumerId);*/ return NULL; } } - tscDebug("consumer:0x%" PRIx64 " handle rsp %p", tmq->consumerId, rspWrapper); - if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__END_RSP) { taosFreeQitem(rspWrapper); terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET; return NULL; } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RSP) { SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)rspWrapper; - tscDebug("consumer:0x%" PRIx64 " actual process poll rsp", tmq->consumerId); + tscDebug("consumer:0x%" PRIx64 " process poll rsp", tmq->consumerId); /*atomic_sub_fetch_32(&tmq->readyRequest, 1);*/ int32_t consumerEpoch = atomic_load_32(&tmq->epoch); if (pollRspWrapper->dataRsp.head.epoch == consumerEpoch) { @@ -1799,6 +1815,9 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP) { SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)rspWrapper; int32_t consumerEpoch = atomic_load_32(&tmq->epoch); + + tscDebug("consumer:0x%" PRIx64 " process meta rsp", tmq->consumerId); + if (pollRspWrapper->metaRsp.head.epoch == consumerEpoch) { SMqClientVg* pVg = pollRspWrapper->vgHandle; /*printf("vgId:%d, offset %" PRId64 " up to %" PRId64 "\n", pVg->vgId, pVg->currentOffset, @@ -1863,7 +1882,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { void* rspObj; int64_t startTime = taosGetTimestampMs(); - tscDebug("consumer:0x%" PRIx64 ", start poll at %" PRId64, tmq->consumerId, startTime); + tscDebug("consumer:0x%" PRIx64 " start to poll at %" PRId64, tmq->consumerId, startTime); #if 0 tmqHandleAllDelayedTask(tmq); @@ -1876,7 +1895,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { // in no topic status, delayed task also need to be processed if (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__INIT) { - tscDebug("consumer:0x%" PRIx64 ", poll return since consumer status is init", tmq->consumerId); + tscDebug("consumer:0x%" PRIx64 " poll return since consumer is init", tmq->consumerId); return NULL; } @@ -1902,25 +1921,25 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { rspObj = tmqHandleAllRsp(tmq, timeout, false); if (rspObj) { - tscDebug("consumer:0x%" PRIx64 ", return rsp %p", tmq->consumerId, rspObj); + tscDebug("consumer:0x%" PRIx64 " return rsp %p", tmq->consumerId, rspObj); return (TAOS_RES*)rspObj; } else if (terrno == TSDB_CODE_TQ_NO_COMMITTED_OFFSET) { - tscDebug("consumer:0x%" PRIx64 ", return null since no committed offset", tmq->consumerId); + tscDebug("consumer:0x%" PRIx64 " return null since no committed offset", tmq->consumerId); return NULL; } if (timeout != -1) { int64_t currentTime = taosGetTimestampMs(); - int64_t passedTime = currentTime - startTime; - if (passedTime > timeout) { - tscDebug("consumer:0x%" PRIx64 ", (epoch %d) timeout, no rsp, start time %" PRId64 ", current time %" PRId64, + int64_t elapsedTime = currentTime - startTime; + if (elapsedTime > timeout) { + tscDebug("consumer:0x%" PRIx64 " (epoch %d) timeout, no rsp, start time %" PRId64 ", current time %" PRId64, tmq->consumerId, tmq->epoch, startTime, currentTime); return NULL; } /*tscInfo("consumer:0x%" PRIx64 ", (epoch %d) wait, start time %" PRId64 ", current time %" PRId64*/ /*", left time %" PRId64,*/ - /*tmq->consumerId, tmq->epoch, startTime, currentTime, (timeout - passedTime));*/ - tsem_timewait(&tmq->rspSem, (timeout - passedTime)); + /*tmq->consumerId, tmq->epoch, startTime, currentTime, (timeout - elapsedTime));*/ + tsem_timewait(&tmq->rspSem, (timeout - elapsedTime)); } else { // use tsem_timewait instead of tsem_wait to avoid unexpected stuck tsem_timewait(&tmq->rspSem, 1000); diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index cb3c2f8c68..59c931d9aa 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -912,7 +912,7 @@ TEST(clientCase, subscription_test) { tmq_conf_t* conf = tmq_conf_new(); tmq_conf_set(conf, "enable.auto.commit", "true"); tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); - tmq_conf_set(conf, "group.id", "newabcdefgjhijlm__"); + tmq_conf_set(conf, "group.id", "cgrpName"); tmq_conf_set(conf, "td.connect.user", "root"); tmq_conf_set(conf, "td.connect.pass", "taosdata"); tmq_conf_set(conf, "auto.offset.reset", "earliest"); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index e866203372..c66d21b516 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -64,13 +64,17 @@ int32_t getJsonValueLen(const char* data) { return dataLen; } -int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull) { +int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull) { + return colDataSetVal(pColumnInfoData, rowIndex, pData, isNull); +} + +int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull) { if (isNull) { // There is a placehold for each NULL value of binary or nchar type. if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { - pColumnInfoData->varmeta.offset[currentRow] = -1; // it is a null value of VAR type. + pColumnInfoData->varmeta.offset[rowIndex] = -1; // it is a null value of VAR type. } else { - colDataSetNull_f_s(pColumnInfoData, currentRow); + colDataSetNull_f_s(pColumnInfoData, rowIndex); } pColumnInfoData->hasNull = true; @@ -95,6 +99,9 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con while (newSize < pAttr->length + dataLen) { newSize = newSize * 1.5; + if (newSize > UINT32_MAX) { + return TSDB_CODE_OUT_OF_MEMORY; + } } char* buf = taosMemoryRealloc(pColumnInfoData->pData, newSize); @@ -107,12 +114,12 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con } uint32_t len = pColumnInfoData->varmeta.length; - pColumnInfoData->varmeta.offset[currentRow] = len; + pColumnInfoData->varmeta.offset[rowIndex] = len; memmove(pColumnInfoData->pData + len, pData, dataLen); pColumnInfoData->varmeta.length += dataLen; } else { - memcpy(pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow, pData, pColumnInfoData->info.bytes); + memcpy(pColumnInfoData->pData + pColumnInfoData->info.bytes * rowIndex, pData, pColumnInfoData->info.bytes); } return 0; @@ -173,7 +180,7 @@ static void doCopyNItems(struct SColumnInfoData* pColumnInfoData, int32_t curren } } -int32_t colDataAppendNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, +int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, uint32_t numOfRows) { int32_t len = pColumnInfoData->info.bytes; if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { @@ -513,10 +520,10 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3 } if (isNull) { - colDataAppendNULL(pDstCol, j - startIndex); + colDataSetNULL(pDstCol, j - startIndex); } else { char* p = colDataGetData(pColData, j); - colDataAppend(pDstCol, j - startIndex, p, false); + colDataSetVal(pDstCol, j - startIndex, p, false); } } } @@ -619,6 +626,33 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) { return TSDB_CODE_SUCCESS; } +static bool colDataIsNNull(const SColumnInfoData* pColumnInfoData, int32_t startIndex, + uint32_t nRows) { + if (!pColumnInfoData->hasNull) { + return false; + } + + if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { + for (int32_t i = startIndex; i < nRows; ++i) { + if (!colDataIsNull_var(pColumnInfoData, i)) { + return false; + } + } + } else { + if (pColumnInfoData->nullbitmap == NULL) { + return false; + } + + for (int32_t i = startIndex; i < nRows; ++i) { + if (!colDataIsNull_f(pColumnInfoData->nullbitmap, i)) { + return false; + } + } + } + + return true; +} + // todo remove this int32_t blockDataFromBuf1(SSDataBlock* pBlock, const char* buf, size_t capacity) { pBlock->info.rows = *(int32_t*)buf; @@ -661,7 +695,7 @@ int32_t blockDataFromBuf1(SSDataBlock* pBlock, const char* buf, size_t capacity) } } - if (!colDataIsNNull_s(pCol, 0, pBlock->info.rows)) { + if (!colDataIsNNull(pCol, 0, pBlock->info.rows)) { memcpy(pCol->pData, pStart, colLength); } @@ -783,13 +817,13 @@ static int32_t doAssignOneTuple(SColumnInfoData* pDstCols, int32_t numOfRows, co SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, i); if (pSrc->hasNull && colDataIsNull(pSrc, pSrcBlock->info.rows, tupleIndex, pSrcBlock->pBlockAgg[i])) { - code = colDataAppend(pDst, numOfRows, NULL, true); + code = colDataSetVal(pDst, numOfRows, NULL, true); if (code != TSDB_CODE_SUCCESS) { return code; } } else { char* p = colDataGetData(pSrc, tupleIndex); - code = colDataAppend(pDst, numOfRows, p, false); + code = colDataSetVal(pDst, numOfRows, p, false); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1413,7 +1447,7 @@ SSDataBlock* blockCopyOneRow(const SSDataBlock* pDataBlock, int32_t rowIdx) { SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i); void* pData = colDataGetData(pSrc, rowIdx); bool isNull = colDataIsNull(pSrc, pDataBlock->info.rows, rowIdx, NULL); - colDataAppend(pDst, 0, pData, isNull); + colDataSetVal(pDst, 0, pData, isNull); } pBlock->info.rows = 1; @@ -1644,7 +1678,7 @@ static void colDataTrimFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_ } } -int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n) { +int32_t blockDataTrimFirstRows(SSDataBlock* pBlock, size_t n) { if (n == 0) { return TSDB_CODE_SUCCESS; } @@ -2528,7 +2562,7 @@ const char* blockDecode(SSDataBlock* pBlock, const char* pData) { pStart += sizeof(uint64_t); if (pBlock->pDataBlock == NULL) { - pBlock->pDataBlock = taosArrayInit_s(numOfCols, sizeof(SColumnInfoData), numOfCols); + pBlock->pDataBlock = taosArrayInit_s(sizeof(SColumnInfoData), numOfCols); } for (int32_t i = 0; i < numOfCols; ++i) { diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index e64646dd03..02d83af3a7 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -61,7 +61,7 @@ int32_t tsHeartbeatInterval = 1000; int32_t tsHeartbeatTimeout = 20 * 1000; // vnode -int64_t tsVndCommitMaxIntervalMs = 60 * 1000; +int64_t tsVndCommitMaxIntervalMs = 600 * 1000; // monitor bool tsEnableMonitor = true; diff --git a/source/common/src/tmisce.c b/source/common/src/tmisce.c index dfb1917fcf..59afce1bbb 100644 --- a/source/common/src/tmisce.c +++ b/source/common/src/tmisce.c @@ -60,6 +60,19 @@ bool isEpsetEqual(const SEpSet* s1, const SEpSet* s2) { return true; } +void epsetAssign(SEpSet* pDst, const SEpSet* pSrc) { + if (pSrc == NULL || pDst == NULL) { + return; + } + + pDst->inUse = pSrc->inUse; + pDst->numOfEps = pSrc->numOfEps; + for (int32_t i = 0; i < pSrc->numOfEps; ++i) { + pDst->eps[i].port = pSrc->eps[i].port; + tstrncpy(pDst->eps[i].fqdn, pSrc->eps[i].fqdn, tListLen(pSrc->eps[i].fqdn)); + } +} + void updateEpSet_s(SCorEpSet* pEpSet, SEpSet* pNewEpSet) { taosCorBeginWrite(&pEpSet->version); pEpSet->epSet = *pNewEpSet; diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index 8460d71e56..8a77087d23 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -109,14 +109,14 @@ TEST(testCase, Datablock_test) { if (i & 0x01) { int32_t len = sprintf(buf, str, i); STR_TO_VARSTR(varbuf, buf) - colDataAppend(p0, i, (const char*)&i, false); - colDataAppend(p1, i, (const char*)varbuf, false); + colDataSetVal(p0, i, (const char*)&i, false); + colDataSetVal(p1, i, (const char*)varbuf, false); memset(varbuf, 0, sizeof(varbuf)); memset(buf, 0, sizeof(buf)); } else { - colDataAppend(p0, i, (const char*)&i, true); - colDataAppend(p1, i, (const char*)varbuf, true); + colDataSetVal(p0, i, (const char*)&i, true); + colDataSetVal(p1, i, (const char*)varbuf, true); } b->info.rows++; @@ -185,8 +185,8 @@ TEST(testCase, non_var_dataBlock_split_test) { SColumnInfoData* p1 = (SColumnInfoData*)taosArrayGet(b->pDataBlock, 1); int8_t v = i; - colDataAppend(p0, i, (const char*)&i, false); - colDataAppend(p1, i, (const char*)&v, false); + colDataSetVal(p0, i, (const char*)&i, false); + colDataSetVal(p1, i, (const char*)&v, false); b->info.rows++; } @@ -231,12 +231,12 @@ TEST(testCase, var_dataBlock_split_test) { SColumnInfoData* p1 = (SColumnInfoData*)taosArrayGet(b->pDataBlock, 1); int8_t v = i; - colDataAppend(p0, i, (const char*)&i, false); + colDataSetVal(p0, i, (const char*)&i, false); sprintf(buf, "the number of row:%d", i); int32_t len = sprintf(buf1, buf, i); STR_TO_VARSTR(buf1, buf) - colDataAppend(p1, i, buf1, false); + colDataSetVal(p1, i, buf1, false); b->info.rows++; memset(buf, 0, sizeof(buf)); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 3e9084bd11..0724fcc63a 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -242,19 +242,19 @@ int32_t dmAppendVariablesToBlock(SSDataBlock *pBlock, int32_t dnodeId) { GRANT_CFG_SKIP; SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, c++); - colDataAppend(pColInfo, i, (const char *)&dnodeId, false); + colDataSetVal(pColInfo, i, (const char *)&dnodeId, false); char name[TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(name, pItem->name, TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE); pColInfo = taosArrayGet(pBlock->pDataBlock, c++); - colDataAppend(pColInfo, i, name, false); + colDataSetVal(pColInfo, i, name, false); char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0}; int32_t valueLen = 0; cfgDumpItemValue(pItem, &value[VARSTR_HEADER_SIZE], TSDB_CONFIG_VALUE_LEN, &valueLen); varDataSetLen(value, valueLen); pColInfo = taosArrayGet(pBlock->pDataBlock, c++); - colDataAppend(pColInfo, i, value, false); + colDataSetVal(pColInfo, i, value, false); numOfRows++; } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 027b9b7d48..c1b3cde9ea 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -166,7 +166,6 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp dGError("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s qtype:%d contLen:%d", pHead->vgId, pMsg, terrstr(), TMSG_INFO(pMsg->msgType), qtype, pHead->contLen); terrno = (terrno != 0) ? terrno : -1; - vmSendResponse(pMsg); return terrno; } diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 785ecc2bf5..d851578a74 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -58,7 +58,7 @@ typedef int32_t (*MndInitFp)(SMnode *pMnode); typedef void (*MndCleanupFp)(SMnode *pMnode); typedef int32_t (*ShowRetrieveFp)(SRpcMsg *pMsg, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); typedef void (*ShowFreeIterFp)(SMnode *pMnode, void *pIter); -typedef struct SQWorker SQHandle; +typedef struct SQueueWorker SQHandle; typedef struct { const char *name; diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index e0d8ecb3eb..94584dfe58 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -278,31 +278,31 @@ static int32_t mndRetrieveClusters(SRpcMsg *pMsg, SShowObj *pShow, SSDataBlock * cols = 0; SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pCluster->id, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pCluster->id, false); char buf[tListLen(pCluster->name) + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(buf, pCluster->name, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, buf, false); + colDataSetVal(pColInfo, numOfRows, buf, false); int32_t upTime = mndGetClusterUpTimeImp(pCluster); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&upTime, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&upTime, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pCluster->createdTime, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pCluster->createdTime, false); char ver[12] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(ver, tsVersionName, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)ver, false); + colDataSetVal(pColInfo, numOfRows, (const char *)ver, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); if (tsExpireTime <= 0) { - colDataAppendNULL(pColInfo, numOfRows); + colDataSetNULL(pColInfo, numOfRows); } else { - colDataAppend(pColInfo, numOfRows, (const char *)&tsExpireTime, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&tsExpireTime, false); } sdbRelease(pSdb, pCluster); diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index eb4fc3cdad..3771cf28c2 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -100,7 +100,7 @@ static int32_t mndProcessConsumerLostMsg(SRpcMsg *pMsg) { return 0; } - mInfo("receive consumer lost msg, consumer id %" PRId64 ", status %s", pLostMsg->consumerId, + mInfo("process consumer lost msg, consumer:0x%" PRIx64 " status:%d(%s)", pLostMsg->consumerId, pConsumer->status, mndConsumerStatusName(pConsumer->status)); if (pConsumer->status != MQ_CONSUMER_STATUS__READY) { @@ -114,9 +114,17 @@ static int32_t mndProcessConsumerLostMsg(SRpcMsg *pMsg) { mndReleaseConsumer(pMnode, pConsumer); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pMsg, "lost-csm"); - if (pTrans == NULL) goto FAIL; - if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto FAIL; - if (mndTransPrepare(pMnode, pTrans) != 0) goto FAIL; + if (pTrans == NULL) { + goto FAIL; + } + + if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) { + goto FAIL; + } + + if (mndTransPrepare(pMnode, pTrans) != 0) { + goto FAIL; + } tDeleteSMqConsumerObj(pConsumerNew); taosMemoryFree(pConsumerNew); @@ -138,8 +146,8 @@ static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) { return -1; } - mInfo("receive consumer recover msg, consumer id %" PRId64 ", status %s", pRecoverMsg->consumerId, - mndConsumerStatusName(pConsumer->status)); + mInfo("receive consumer recover msg, consumer:0x%" PRIx64 " status:%d(%s)", pRecoverMsg->consumerId, + pConsumer->status, mndConsumerStatusName(pConsumer->status)); if (pConsumer->status != MQ_CONSUMER_STATUS__LOST_REBD) { mndReleaseConsumer(pMnode, pConsumer); @@ -153,7 +161,10 @@ static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) { mndReleaseConsumer(pMnode, pConsumer); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "recover-csm"); - if (pTrans == NULL) goto FAIL; + if (pTrans == NULL) { + goto FAIL; + } + if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto FAIL; if (mndTransPrepare(pMnode, pTrans) != 0) goto FAIL; @@ -241,25 +252,30 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { // iterate all consumers, find all modification while (1) { pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer); - if (pIter == NULL) break; + if (pIter == NULL) { + break; + } int32_t hbStatus = atomic_add_fetch_32(&pConsumer->hbStatus, 1); int32_t status = atomic_load_32(&pConsumer->status); - if (status == MQ_CONSUMER_STATUS__READY && hbStatus > MND_CONSUMER_LOST_HB_CNT) { - SMqConsumerLostMsg *pLostMsg = rpcMallocCont(sizeof(SMqConsumerLostMsg)); - pLostMsg->consumerId = pConsumer->consumerId; - SRpcMsg rpcMsg = { - .msgType = TDMT_MND_TMQ_CONSUMER_LOST, - .pCont = pLostMsg, - .contLen = sizeof(SMqConsumerLostMsg), - }; - tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); - } + mDebug("check for consumer:0x%" PRIx64 " status:%d(%s), sub-time:%" PRId64 ", uptime:%" PRId64, + pConsumer->consumerId, status, mndConsumerStatusName(status), pConsumer->subscribeTime, pConsumer->upTime); if (status == MQ_CONSUMER_STATUS__READY) { - // do nothing + if (hbStatus > MND_CONSUMER_LOST_HB_CNT) { + SMqConsumerLostMsg *pLostMsg = rpcMallocCont(sizeof(SMqConsumerLostMsg)); + + pLostMsg->consumerId = pConsumer->consumerId; + SRpcMsg rpcMsg = { + .msgType = TDMT_MND_TMQ_CONSUMER_LOST, + .pCont = pLostMsg, + .contLen = sizeof(SMqConsumerLostMsg), + }; + tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); + } } else if (status == MQ_CONSUMER_STATUS__LOST_REBD) { + // if the client is lost longer than one day, clear it. Otherwise, do nothing about the lost consumers. if (hbStatus > MND_CONSUMER_LOST_CLEAR_THRESHOLD) { SMqConsumerClearMsg *pClearMsg = rpcMallocCont(sizeof(SMqConsumerClearMsg)); @@ -338,7 +354,7 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) { int64_t consumerId = req.consumerId; SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, consumerId); if (pConsumer == NULL) { - mError("consumer %" PRId64 " not exist", consumerId); + mError("consumer:0x%" PRIx64 " not exist", consumerId); terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST; return -1; } @@ -348,7 +364,7 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) { int32_t status = atomic_load_32(&pConsumer->status); if (status == MQ_CONSUMER_STATUS__LOST_REBD) { - mInfo("try to recover consumer %" PRId64 "", consumerId); + mInfo("try to recover consumer:0x%" PRIx64 "", consumerId); SMqConsumerRecoverMsg *pRecoverMsg = rpcMallocCont(sizeof(SMqConsumerRecoverMsg)); pRecoverMsg->consumerId = consumerId; @@ -380,6 +396,15 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, consumerId); if (pConsumer == NULL) { + mError("consumer:0x%" PRIx64 " group:%s not exists in sdb", consumerId, req.cgroup); + terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST; + return -1; + } + + int32_t ret = strncmp(req.cgroup, pConsumer->cgroup, tListLen(pConsumer->cgroup)); + if (ret != 0) { + mError("consumer:0x%" PRIx64 " group:%s not consistent with data in sdb, saved cgroup:%s", consumerId, req.cgroup, + pConsumer->cgroup); terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST; return -1; } @@ -391,7 +416,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { #if 1 if (status == MQ_CONSUMER_STATUS__LOST_REBD) { - mInfo("try to recover consumer %" PRId64 "", consumerId); + mInfo("try to recover consumer:0x%" PRIx64, consumerId); SMqConsumerRecoverMsg *pRecoverMsg = rpcMallocCont(sizeof(SMqConsumerRecoverMsg)); pRecoverMsg->consumerId = consumerId; @@ -400,22 +425,24 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { .pCont = pRecoverMsg, .contLen = sizeof(SMqConsumerRecoverMsg), }; + tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &pRpcMsg); } #endif if (status != MQ_CONSUMER_STATUS__READY) { - mInfo("consumer %" PRId64 " not ready, status: %s", consumerId, mndConsumerStatusName(status)); + mInfo("consumer:0x%" PRIx64 " not ready, status: %s", consumerId, mndConsumerStatusName(status)); terrno = TSDB_CODE_MND_CONSUMER_NOT_READY; return -1; } int32_t serverEpoch = atomic_load_32(&pConsumer->epoch); - // 2. check epoch, only send ep info when epoches do not match + // 2. check epoch, only send ep info when epochs do not match if (epoch != serverEpoch) { taosRLockLatch(&pConsumer->lock); - mInfo("process ask ep, consumer:%" PRId64 "(epoch %d), server epoch %d", consumerId, epoch, serverEpoch); + mInfo("process ask ep, consumer:0x%" PRIx64 "(epoch %d) update with server epoch %d", consumerId, epoch, + serverEpoch); int32_t numOfTopics = taosArrayGetSize(pConsumer->currentTopics); rsp.topics = taosArrayInit(numOfTopics, sizeof(SMqSubTopicEp)); @@ -425,7 +452,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { goto FAIL; } - // handle all topic subscribed by the consumer + // handle all topics subscribed by this consumer for (int32_t i = 0; i < numOfTopics; i++) { char *topic = taosArrayGetP(pConsumer->currentTopics, i); SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, topic); @@ -452,6 +479,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { SMqConsumerEp *pConsumerEp = taosHashGet(pSub->consumerHash, &consumerId, sizeof(int64_t)); int32_t vgNum = taosArrayGetSize(pConsumerEp->vgs); + // this customer assigned vgroups topicEp.vgs = taosArrayInit(vgNum, sizeof(SMqSubVgEp)); if (topicEp.vgs == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -481,6 +509,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { } taosRUnLockLatch(&pConsumer->lock); } + // encode rsp int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqAskEpRsp(NULL, &rsp); void *buf = rpcMallocCont(tlen); @@ -488,6 +517,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } + ((SMqRspHead *)buf)->mqMsgType = TMQ_MSG_TYPE__EP_RSP; ((SMqRspHead *)buf)->epoch = serverEpoch; ((SMqRspHead *)buf)->consumerId = pConsumer->consumerId; @@ -503,6 +533,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { pMsg->info.rsp = buf; pMsg->info.rspLen = tlen; return 0; + FAIL: tDeleteSMqAskEpRsp(&rsp); mndReleaseConsumer(pMnode, pConsumer); @@ -525,12 +556,14 @@ int32_t mndSetConsumerCommitLogs(SMnode *pMnode, STrans *pTrans, SMqConsumerObj return 0; } -static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { - SMnode *pMnode = pMsg->info.node; - char *msgStr = pMsg->pCont; +int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { + SMnode *pMnode = pMsg->info.node; + char *msgStr = pMsg->pCont; + SCMSubscribeReq subscribe = {0}; tDeserializeSCMSubscribeReq(msgStr, &subscribe); - int64_t consumerId = subscribe.consumerId; + + uint64_t consumerId = subscribe.consumerId; char *cgroup = subscribe.cgroup; SMqConsumerObj *pConsumerOld = NULL; SMqConsumerObj *pConsumerNew = NULL; @@ -541,25 +574,27 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { taosArrayRemoveDuplicateP(newSub, taosArrayCompareString, taosMemoryFree); int32_t newTopicNum = taosArrayGetSize(newSub); - // check topic existance + + // check topic existence STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "subscribe"); - if (pTrans == NULL) goto SUBSCRIBE_OVER; + if (pTrans == NULL) { + goto _over; + } for (int32_t i = 0; i < newTopicNum; i++) { char *topic = taosArrayGetP(newSub, i); SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); - if (pTopic == NULL) { - terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST; - goto SUBSCRIBE_OVER; + if (pTopic == NULL) { // terrno has been set by callee function + goto _over; } if (mndCheckTopicPrivilege(pMnode, pMsg->info.conn.user, MND_OPER_SUBSCRIBE, pTopic) != 0) { mndReleaseTopic(pMnode, pTopic); - goto SUBSCRIBE_OVER; + goto _over; } if (mndCheckTopicPrivilege(pMnode, pMsg->info.conn.user, MND_OPER_SUBSCRIBE, pTopic) != 0) { - goto SUBSCRIBE_OVER; + goto _over; } mndReleaseTopic(pMnode, pTopic); @@ -581,8 +616,8 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { taosArrayPush(pConsumerNew->assignedTopics, &newTopicCopy); } - if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto SUBSCRIBE_OVER; - if (mndTransPrepare(pMnode, pTrans) != 0) goto SUBSCRIBE_OVER; + if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto _over; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _over; } else { /*taosRLockLatch(&pConsumerOld->lock);*/ @@ -594,13 +629,13 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { if (status != MQ_CONSUMER_STATUS__READY) { terrno = TSDB_CODE_MND_CONSUMER_NOT_READY; - goto SUBSCRIBE_OVER; + goto _over; } pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup); if (pConsumerNew == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - goto SUBSCRIBE_OVER; + goto _over; } pConsumerNew->updateType = CONSUMER_UPDATE__MODIFY; @@ -653,16 +688,16 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { /*if (taosArrayGetSize(pConsumerNew->assignedTopics) == 0) {*/ /*pConsumerNew->updateType = */ /*}*/ - goto SUBSCRIBE_OVER; + goto _over; } - if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto SUBSCRIBE_OVER; - if (mndTransPrepare(pMnode, pTrans) != 0) goto SUBSCRIBE_OVER; + if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto _over; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _over; } code = TSDB_CODE_ACTION_IN_PROGRESS; -SUBSCRIBE_OVER: +_over: mndTransDrop(pTrans); if (pConsumerOld) { @@ -715,13 +750,14 @@ CM_ENCODE_OVER: } SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { - terrno = TSDB_CODE_OUT_OF_MEMORY; SSdbRow *pRow = NULL; SMqConsumerObj *pConsumer = NULL; void *buf = NULL; int8_t sver = 0; - if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto CM_DECODE_OVER; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) { + goto CM_DECODE_OVER; + } if (sver != MND_CONSUMER_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; @@ -729,52 +765,63 @@ SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { } pRow = sdbAllocRow(sizeof(SMqConsumerObj)); - if (pRow == NULL) goto CM_DECODE_OVER; + if (pRow == NULL) { + goto CM_DECODE_OVER; + } pConsumer = sdbGetRowObj(pRow); - if (pConsumer == NULL) goto CM_DECODE_OVER; + if (pConsumer == NULL) { + goto CM_DECODE_OVER; + } int32_t dataPos = 0; int32_t len; SDB_GET_INT32(pRaw, dataPos, &len, CM_DECODE_OVER); buf = taosMemoryMalloc(len); - if (buf == NULL) goto CM_DECODE_OVER; + if (buf == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto CM_DECODE_OVER; + } + SDB_GET_BINARY(pRaw, dataPos, buf, len, CM_DECODE_OVER); SDB_GET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE, CM_DECODE_OVER); if (tDecodeSMqConsumerObj(buf, pConsumer) == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; // TODO set correct error code goto CM_DECODE_OVER; } - tmsgUpdateDnodeEpSet(&pConsumer->ep); - terrno = TSDB_CODE_SUCCESS; + tmsgUpdateDnodeEpSet(&pConsumer->ep); CM_DECODE_OVER: taosMemoryFreeClear(buf); if (terrno != TSDB_CODE_SUCCESS) { - mError("consumer:%" PRId64 ", failed to decode from raw:%p since %s", pConsumer == NULL ? 0 : pConsumer->consumerId, - pRaw, terrstr()); + mError("consumer:0x%" PRIx64 " failed to decode from raw:%p since %s", + pConsumer == NULL ? 0 : pConsumer->consumerId, pRaw, terrstr()); taosMemoryFreeClear(pRow); - return NULL; } return pRow; } static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer) { - mTrace("consumer:%" PRId64 ", perform insert action", pConsumer->consumerId); + mDebug("consumer:0x%" PRIx64 " cgroup:%s status:%d(%s) epoch:%d load from sdb, perform insert action", + pConsumer->consumerId, pConsumer->cgroup, pConsumer->status, mndConsumerStatusName(pConsumer->status), + pConsumer->epoch); pConsumer->subscribeTime = pConsumer->upTime; return 0; } static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer) { - mTrace("consumer:%" PRId64 ", perform delete action", pConsumer->consumerId); + mDebug("consumer:0x%" PRIx64 " perform delete action, status:%s", pConsumer->consumerId, + mndConsumerStatusName(pConsumer->status)); tDeleteSMqConsumerObj(pConsumer); return 0; } static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, SMqConsumerObj *pNewConsumer) { - mTrace("consumer:%" PRId64 ", perform update action", pOldConsumer->consumerId); + mDebug("consumer:0x%" PRIx64 " perform update action, update type:%d, subscribe-time:%" PRId64 ", uptime:%" PRId64, + pOldConsumer->consumerId, pNewConsumer->updateType, pOldConsumer->subscribeTime, pOldConsumer->upTime); taosWLockLatch(&pOldConsumer->lock); @@ -814,7 +861,11 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, pOldConsumer->rebalanceTime = pNewConsumer->upTime; + int32_t status = pOldConsumer->status; pOldConsumer->status = MQ_CONSUMER_STATUS__LOST; + mDebug("consumer:0x%" PRIx64 " state %s -> %s, reb-time:%" PRId64 ", reb-removed-topics:%d", + pOldConsumer->consumerId, mndConsumerStatusName(status), mndConsumerStatusName(pOldConsumer->status), + pOldConsumer->rebalanceTime, (int)taosArrayGetSize(pOldConsumer->rebRemovedTopics)); } else if (pNewConsumer->updateType == CONSUMER_UPDATE__RECOVER) { /*A(taosArrayGetSize(pOldConsumer->currentTopics) == 0);*/ /*A(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0);*/ @@ -842,7 +893,8 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, bool existing = false; #if 1 - for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->currentTopics); i++) { + int32_t numOfExistedTopics = taosArrayGetSize(pOldConsumer->currentTopics); + for (int32_t i = 0; i < numOfExistedTopics; i++) { char *topic = taosArrayGetP(pOldConsumer->currentTopics, i); if (strcmp(topic, addedTopic) == 0) { existing = true; @@ -867,27 +919,28 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, } // set status + int32_t status = pOldConsumer->status; if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0) { - if (pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY || - pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) { + if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) { pOldConsumer->status = MQ_CONSUMER_STATUS__READY; - } else if (pOldConsumer->status == MQ_CONSUMER_STATUS__LOST_IN_REB || - pOldConsumer->status == MQ_CONSUMER_STATUS__LOST) { + } else if (status == MQ_CONSUMER_STATUS__LOST_IN_REB || status == MQ_CONSUMER_STATUS__LOST) { pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_REBD; } } else { - if (pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY || - pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) { + if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) { pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY_IN_REB; - } else if (pOldConsumer->status == MQ_CONSUMER_STATUS__LOST || - pOldConsumer->status == MQ_CONSUMER_STATUS__LOST_IN_REB) { + } else if (status == MQ_CONSUMER_STATUS__LOST || status == MQ_CONSUMER_STATUS__LOST_IN_REB) { pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_IN_REB; } } + // the re-balance is triggered when the new consumer is launched. pOldConsumer->rebalanceTime = pNewConsumer->upTime; atomic_add_fetch_32(&pOldConsumer->epoch, 1); + mDebug("consumer:0x%" PRIx64 " state %s -> %s, new epoch:%d, reb-time:%" PRId64 ", current topics:%d", + pOldConsumer->consumerId, mndConsumerStatusName(status), mndConsumerStatusName(pOldConsumer->status), + pOldConsumer->epoch, pOldConsumer->rebalanceTime, (int)taosArrayGetSize(pOldConsumer->currentTopics)); } else if (pNewConsumer->updateType == CONSUMER_UPDATE__REMOVE) { /*A(taosArrayGetSize(pNewConsumer->rebNewTopics) == 0);*/ /*A(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 1);*/ @@ -926,27 +979,27 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, /*A(i < sz);*/ // set status + int32_t status = pOldConsumer->status; if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0) { - if (pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY || - pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) { + if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) { pOldConsumer->status = MQ_CONSUMER_STATUS__READY; - } else if (pOldConsumer->status == MQ_CONSUMER_STATUS__LOST_IN_REB || - pOldConsumer->status == MQ_CONSUMER_STATUS__LOST) { + } else if (status == MQ_CONSUMER_STATUS__LOST_IN_REB || status == MQ_CONSUMER_STATUS__LOST) { pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_REBD; } } else { - if (pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY || - pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) { + if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) { pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY_IN_REB; - } else if (pOldConsumer->status == MQ_CONSUMER_STATUS__LOST || - pOldConsumer->status == MQ_CONSUMER_STATUS__LOST_IN_REB) { + } else if (status == MQ_CONSUMER_STATUS__LOST || status == MQ_CONSUMER_STATUS__LOST_IN_REB) { pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_IN_REB; } } pOldConsumer->rebalanceTime = pNewConsumer->upTime; - atomic_add_fetch_32(&pOldConsumer->epoch, 1); + + mDebug("consumer:0x%" PRIx64 " state %s -> %s, new epoch:%d, reb-time:%" PRId64 ", current topics:%d", + pOldConsumer->consumerId, mndConsumerStatusName(status), mndConsumerStatusName(pOldConsumer->status), + pOldConsumer->epoch, pOldConsumer->rebalanceTime, (int)taosArrayGetSize(pOldConsumer->currentTopics)); } taosWUnLockLatch(&pOldConsumer->lock); @@ -975,16 +1028,19 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock * while (numOfRows < rowsCapacity) { pShow->pIter = sdbFetch(pSdb, SDB_CONSUMER, pShow->pIter, (void **)&pConsumer); - if (pShow->pIter == NULL) break; + if (pShow->pIter == NULL) { + break; + } + if (taosArrayGetSize(pConsumer->assignedTopics) == 0) { - mDebug("showing consumer %" PRId64 " no assigned topic, skip", pConsumer->consumerId); + mDebug("showing consumer:0x%" PRIx64 " no assigned topic, skip", pConsumer->consumerId); sdbRelease(pSdb, pConsumer); continue; } taosRLockLatch(&pConsumer->lock); - mDebug("showing consumer %" PRId64, pConsumer->consumerId); + mDebug("showing consumer:0x%" PRIx64, pConsumer->consumerId); int32_t topicSz = taosArrayGetSize(pConsumer->assignedTopics); bool hasTopic = true; @@ -1003,56 +1059,55 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock * // consumer id pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->consumerId, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->consumerId, false); // consumer group char cgroup[TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE] = {0}; tstrncpy(varDataVal(cgroup), pConsumer->cgroup, TSDB_CGROUP_LEN); varDataSetLen(cgroup, strlen(varDataVal(cgroup))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)cgroup, false); + colDataSetVal(pColInfo, numOfRows, (const char *)cgroup, false); // client id char clientId[256 + VARSTR_HEADER_SIZE] = {0}; tstrncpy(varDataVal(clientId), pConsumer->clientId, 256); varDataSetLen(clientId, strlen(varDataVal(clientId))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)clientId, false); + colDataSetVal(pColInfo, numOfRows, (const char *)clientId, false); // status char status[20 + VARSTR_HEADER_SIZE] = {0}; tstrncpy(varDataVal(status), mndConsumerStatusName(pConsumer->status), 20); varDataSetLen(status, strlen(varDataVal(status))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)status, false); + colDataSetVal(pColInfo, numOfRows, (const char *)status, false); // one subscribed topic pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); if (hasTopic) { char topic[TSDB_TOPIC_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; const char *topicName = mndTopicGetShowName(taosArrayGetP(pConsumer->assignedTopics, i)); - tstrncpy(varDataVal(topic), topicName, TSDB_TOPIC_FNAME_LEN); - varDataSetLen(topic, strlen(varDataVal(topic))); - colDataAppend(pColInfo, numOfRows, (const char *)topic, false); + STR_TO_VARSTR(topic, topicName); + colDataSetVal(pColInfo, numOfRows, (const char *)topic, false); } else { - colDataAppend(pColInfo, numOfRows, NULL, true); + colDataSetVal(pColInfo, numOfRows, NULL, true); } // end point /*pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);*/ - /*colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->ep, true);*/ + /*colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->ep, true);*/ // up time pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->upTime, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->upTime, false); // subscribe time pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->subscribeTime, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->subscribeTime, false); // rebalance time pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->rebalanceTime, pConsumer->rebalanceTime == 0); + colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->rebalanceTime, pConsumer->rebalanceTime == 0); numOfRows++; } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 2f7fa26a87..d2b16967f9 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1703,46 +1703,46 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, for (int32_t i = 0; i < pShow->numOfColumns; ++i) { SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, i); if (i == 0) { - colDataAppend(pColInfo, rows, buf, false); + colDataSetVal(pColInfo, rows, buf, false); } else if (i == 1) { - colDataAppend(pColInfo, rows, (const char *)&pDb->createdTime, false); + colDataSetVal(pColInfo, rows, (const char *)&pDb->createdTime, false); } else if (i == 3) { - colDataAppend(pColInfo, rows, (const char *)&numOfTables, false); + colDataSetVal(pColInfo, rows, (const char *)&numOfTables, false); } else if (i == 14) { - colDataAppend(pColInfo, rows, precVstr, false); + colDataSetVal(pColInfo, rows, precVstr, false); } else if (i == 15) { - colDataAppend(pColInfo, rows, statusVstr, false); + colDataSetVal(pColInfo, rows, statusVstr, false); } else { - colDataAppendNULL(pColInfo, rows); + colDataSetNULL(pColInfo, rows); } } } else { SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, buf, false); + colDataSetVal(pColInfo, rows, buf, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->createdTime, false); + colDataSetVal(pColInfo, rows, (const char *)&pDb->createdTime, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.numOfVgroups, false); + colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.numOfVgroups, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&numOfTables, false); + colDataSetVal(pColInfo, rows, (const char *)&numOfTables, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.replications, false); + colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.replications, false); const char *strictStr = pDb->cfg.strict ? "on" : "off"; char strictVstr[24] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(strictVstr, strictStr, 24); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)strictVstr, false); + colDataSetVal(pColInfo, rows, (const char *)strictVstr, false); char durationVstr[128] = {0}; int32_t len = sprintf(&durationVstr[VARSTR_HEADER_SIZE], "%dm", pDb->cfg.daysPerFile); varDataSetLen(durationVstr, len); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)durationVstr, false); + colDataSetVal(pColInfo, rows, (const char *)durationVstr, false); char keepVstr[128] = {0}; if (pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep1 || pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep2) { @@ -1754,86 +1754,86 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, } varDataSetLen(keepVstr, len); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)keepVstr, false); + colDataSetVal(pColInfo, rows, (const char *)keepVstr, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.buffer, false); + colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.buffer, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.pageSize, false); + colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.pageSize, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.pages, false); + colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.pages, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.minRows, false); + colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.minRows, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.maxRows, false); + colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.maxRows, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.compression, false); + colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.compression, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)precVstr, false); + colDataSetVal(pColInfo, rows, (const char *)precVstr, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)statusVstr, false); + colDataSetVal(pColInfo, rows, (const char *)statusVstr, false); char *rentensionVstr = buildRetension(pDb->cfg.pRetensions); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); if (rentensionVstr == NULL) { - colDataAppendNULL(pColInfo, rows); + colDataSetNULL(pColInfo, rows); } else { - colDataAppend(pColInfo, rows, (const char *)rentensionVstr, false); + colDataSetVal(pColInfo, rows, (const char *)rentensionVstr, false); taosMemoryFree(rentensionVstr); } pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.numOfStables, false); + colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.numOfStables, false); const char *cacheModelStr = getCacheModelStr(pDb->cfg.cacheLast); char cacheModelVstr[24] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(cacheModelVstr, cacheModelStr, 24); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)cacheModelVstr, false); + colDataSetVal(pColInfo, rows, (const char *)cacheModelVstr, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.cacheLastSize, false); + colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.cacheLastSize, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.walLevel, false); + colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.walLevel, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.walFsyncPeriod, false); + colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.walFsyncPeriod, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.walRetentionPeriod, false); + colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.walRetentionPeriod, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.walRetentionSize, false); + colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.walRetentionSize, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.walRollPeriod, false); + colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.walRollPeriod, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.walSegmentSize, false); + colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.walSegmentSize, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.sstTrigger, false); + colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.sstTrigger, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); int16_t hashPrefix = pDb->cfg.hashPrefix; if (hashPrefix > 0) { hashPrefix = pDb->cfg.hashPrefix - strlen(pDb->name) - 1; } - colDataAppend(pColInfo, rows, (const char *)&hashPrefix, false); + colDataSetVal(pColInfo, rows, (const char *)&hashPrefix, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.hashSuffix, false); + colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.hashSuffix, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.tsdbPageSize, false); + colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.tsdbPageSize, false); } taosMemoryFree(buf); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 7f95774ea0..5dd1742e5e 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -1007,11 +1007,11 @@ static int32_t mndRetrieveConfigs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p STR_WITH_MAXSIZE_TO_VARSTR(buf, cfgOpts[i], TSDB_CONFIG_OPTION_LEN); SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)buf, false); + colDataSetVal(pColInfo, numOfRows, (const char *)buf, false); STR_WITH_MAXSIZE_TO_VARSTR(bufVal, cfgVals[i], TSDB_CONFIG_VALUE_LEN); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)bufVal, false); + colDataSetVal(pColInfo, numOfRows, (const char *)bufVal, false); numOfRows++; } @@ -1039,20 +1039,20 @@ static int32_t mndRetrieveDnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB cols = 0; SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pDnode->id, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pDnode->id, false); char buf[tListLen(pDnode->ep) + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(buf, pDnode->ep, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, buf, false); + colDataSetVal(pColInfo, numOfRows, buf, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); int16_t id = mndGetVnodesNum(pMnode, pDnode->id); - colDataAppend(pColInfo, numOfRows, (const char *)&id, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&id, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pDnode->numOfSupportVnodes, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pDnode->numOfSupportVnodes, false); const char *status = "ready"; if (objStatus == SDB_STATUS_CREATING) status = "creating"; @@ -1069,16 +1069,16 @@ static int32_t mndRetrieveDnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB char b1[16] = {0}; STR_TO_VARSTR(b1, status); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, b1, false); + colDataSetVal(pColInfo, numOfRows, b1, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pDnode->createdTime, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pDnode->createdTime, false); char *b = taosMemoryCalloc(VARSTR_HEADER_SIZE + strlen(offlineReason[pDnode->offlineReason]) + 1, 1); STR_TO_VARSTR(b, online ? "" : offlineReason[pDnode->offlineReason]); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, b, false); + colDataSetVal(pColInfo, numOfRows, b, false); taosMemoryFreeClear(b); numOfRows++; diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 244e6058d4..8d006f1029 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -511,39 +511,39 @@ static int32_t mndRetrieveFuncs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl STR_WITH_MAXSIZE_TO_VARSTR(b1, pFunc->name, pShow->pMeta->pSchemas[cols].bytes); SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)b1, false); + colDataSetVal(pColInfo, numOfRows, (const char *)b1, false); if (pFunc->pComment) { char *b2 = taosMemoryCalloc(1, pShow->pMeta->pSchemas[cols].bytes); STR_WITH_MAXSIZE_TO_VARSTR(b2, pFunc->pComment, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)b2, false); + colDataSetVal(pColInfo, numOfRows, (const char *)b2, false); } else { pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, NULL, true); + colDataSetVal(pColInfo, numOfRows, NULL, true); } int32_t isAgg = (pFunc->funcType == TSDB_FUNC_TYPE_AGGREGATE) ? 1 : 0; pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&isAgg, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&isAgg, false); char b3[TSDB_TYPE_STR_MAX_LEN + 1] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(b3, mnodeGenTypeStr(buf, TSDB_TYPE_STR_MAX_LEN, pFunc->outputType, pFunc->outputLen), pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)b3, false); + colDataSetVal(pColInfo, numOfRows, (const char *)b3, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pFunc->createdTime, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pFunc->createdTime, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pFunc->codeSize, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pFunc->codeSize, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pFunc->bufSize, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pFunc->bufSize, false); numOfRows++; sdbRelease(pSdb, pFunc); diff --git a/source/dnode/mnode/impl/src/mndGrant.c b/source/dnode/mnode/impl/src/mndGrant.c index 42851bd16a..1b46e16961 100644 --- a/source/dnode/mnode/impl/src/mndGrant.c +++ b/source/dnode/mnode/impl/src/mndGrant.c @@ -29,85 +29,85 @@ static int32_t mndRetrieveGrant(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols); const char *src = "community"; STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); - colDataAppend(pColInfo, numOfRows, tmp, false); + colDataSetVal(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "unlimited"; STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); - colDataAppend(pColInfo, numOfRows, tmp, false); + colDataSetVal(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "false"; STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); - colDataAppend(pColInfo, numOfRows, tmp, false); + colDataSetVal(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "unlimited"; STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); - colDataAppend(pColInfo, numOfRows, tmp, false); + colDataSetVal(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "unlimited"; STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); - colDataAppend(pColInfo, numOfRows, tmp, false); + colDataSetVal(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "unlimited"; STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); - colDataAppend(pColInfo, numOfRows, tmp, false); + colDataSetVal(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "unlimited"; STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); - colDataAppend(pColInfo, numOfRows, tmp, false); + colDataSetVal(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "unlimited"; STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); - colDataAppend(pColInfo, numOfRows, tmp, false); + colDataSetVal(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "unlimited"; STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); - colDataAppend(pColInfo, numOfRows, tmp, false); + colDataSetVal(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "unlimited"; STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); - colDataAppend(pColInfo, numOfRows, tmp, false); + colDataSetVal(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "unlimited"; STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); - colDataAppend(pColInfo, numOfRows, tmp, false); + colDataSetVal(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "unlimited"; STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); - colDataAppend(pColInfo, numOfRows, tmp, false); + colDataSetVal(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "unlimited"; STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); - colDataAppend(pColInfo, numOfRows, tmp, false); + colDataSetVal(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "unlimited"; STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); - colDataAppend(pColInfo, numOfRows, tmp, false); + colDataSetVal(pColInfo, numOfRows, tmp, false); numOfRows++; } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index add32fd335..aada00296e 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -608,13 +608,13 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB cols = 0; SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pObj->id, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pObj->id, false); char b1[TSDB_EP_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(b1, pObj->pDnode->ep, TSDB_EP_LEN + VARSTR_HEADER_SIZE); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, b1, false); + colDataSetVal(pColInfo, numOfRows, b1, false); char role[20] = "offline"; if (pObj->id == pMnode->selfDnodeId) { @@ -630,7 +630,7 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB char b2[12 + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(b2, role, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)b2, false); + colDataSetVal(pColInfo, numOfRows, (const char *)b2, false); const char *status = "ready"; if (objStatus == SDB_STATUS_CREATING) status = "creating"; @@ -639,10 +639,10 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB char b3[9 + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(b3, status, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)b3, false); + colDataSetVal(pColInfo, numOfRows, (const char *)b3, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pObj->createdTime, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pObj->createdTime, false); numOfRows++; sdbRelease(pSdb, pObj); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index ffc357b2e8..41dc57f32e 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -709,32 +709,32 @@ static int32_t mndRetrieveConns(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl cols = 0; SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pConn->id, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pConn->id, false); char user[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(user, pConn->user); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)user, false); + colDataSetVal(pColInfo, numOfRows, (const char *)user, false); char app[TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE]; STR_TO_VARSTR(app, pConn->app); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)app, false); + colDataSetVal(pColInfo, numOfRows, (const char *)app, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pConn->pid, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pConn->pid, false); char endpoint[TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE] = {0}; sprintf(&endpoint[VARSTR_HEADER_SIZE], "%s:%d", taosIpStr(pConn->ip), pConn->port); varDataLen(endpoint) = strlen(&endpoint[VARSTR_HEADER_SIZE]); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)endpoint, false); + colDataSetVal(pColInfo, numOfRows, (const char *)endpoint, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pConn->loginTimeMs, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pConn->loginTimeMs, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pConn->lastAccessTimeMs, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pConn->lastAccessTimeMs, false); numOfRows++; } @@ -777,44 +777,44 @@ static int32_t mndRetrieveQueries(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p sprintf(&queryId[VARSTR_HEADER_SIZE], "%x:%" PRIx64, pConn->id, pQuery->reqRid); varDataLen(queryId) = strlen(&queryId[VARSTR_HEADER_SIZE]); SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)queryId, false); + colDataSetVal(pColInfo, numOfRows, (const char *)queryId, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pQuery->queryId, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pQuery->queryId, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pConn->id, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pConn->id, false); char app[TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE]; STR_TO_VARSTR(app, pConn->app); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)app, false); + colDataSetVal(pColInfo, numOfRows, (const char *)app, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pConn->pid, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pConn->pid, false); char user[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(user, pConn->user); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)user, false); + colDataSetVal(pColInfo, numOfRows, (const char *)user, false); char endpoint[TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE] = {0}; sprintf(&endpoint[VARSTR_HEADER_SIZE], "%s:%d", taosIpStr(pConn->ip), pConn->port); varDataLen(endpoint) = strlen(&endpoint[VARSTR_HEADER_SIZE]); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)endpoint, false); + colDataSetVal(pColInfo, numOfRows, (const char *)endpoint, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pQuery->stime, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pQuery->stime, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pQuery->useconds, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pQuery->useconds, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pQuery->stableQuery, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pQuery->stableQuery, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pQuery->subPlanNum, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pQuery->subPlanNum, false); char subStatus[TSDB_SHOW_SUBQUERY_LEN + VARSTR_HEADER_SIZE] = {0}; int32_t strSize = sizeof(subStatus); @@ -828,12 +828,12 @@ static int32_t mndRetrieveQueries(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p } varDataLen(subStatus) = strlen(&subStatus[VARSTR_HEADER_SIZE]); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, subStatus, false); + colDataSetVal(pColInfo, numOfRows, subStatus, false); char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(sql, pQuery->sql); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)sql, false); + colDataSetVal(pColInfo, numOfRows, (const char *)sql, false); numOfRows++; } @@ -867,55 +867,55 @@ static int32_t mndRetrieveApps(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlo cols = 0; SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pApp->appId, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pApp->appId, false); char ip[TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE] = {0}; sprintf(&ip[VARSTR_HEADER_SIZE], "%s", taosIpStr(pApp->ip)); varDataLen(ip) = strlen(&ip[VARSTR_HEADER_SIZE]); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)ip, false); + colDataSetVal(pColInfo, numOfRows, (const char *)ip, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pApp->pid, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pApp->pid, false); char name[TSDB_APP_NAME_LEN + 6 + VARSTR_HEADER_SIZE] = {0}; sprintf(&name[VARSTR_HEADER_SIZE], "%s", pApp->name); varDataLen(name) = strlen(&name[VARSTR_HEADER_SIZE]); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)name, false); + colDataSetVal(pColInfo, numOfRows, (const char *)name, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pApp->startTime, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pApp->startTime, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pApp->summary.numOfInsertsReq, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pApp->summary.numOfInsertsReq, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pApp->summary.numOfInsertRows, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pApp->summary.numOfInsertRows, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pApp->summary.insertElapsedTime, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pApp->summary.insertElapsedTime, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pApp->summary.insertBytes, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pApp->summary.insertBytes, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pApp->summary.fetchBytes, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pApp->summary.fetchBytes, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pApp->summary.queryElapsedTime, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pApp->summary.queryElapsedTime, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pApp->summary.numOfSlowQueries, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pApp->summary.numOfSlowQueries, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pApp->summary.totalRequests, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pApp->summary.totalRequests, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pApp->summary.currentRequests, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pApp->summary.currentRequests, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pApp->lastAccessTimeMs, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pApp->lastAccessTimeMs, false); numOfRows++; } diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 28a5dee2db..a8b2d5f4bb 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -507,15 +507,15 @@ static int32_t mndRetrieveQnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB cols = 0; SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pObj->id, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pObj->id, false); char ep[TSDB_EP_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(ep, pObj->pDnode->ep, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)ep, false); + colDataSetVal(pColInfo, numOfRows, (const char *)ep, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pObj->createdTime, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pObj->createdTime, false); numOfRows++; sdbRelease(pSdb, pObj); diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index bdef8000bd..a5fed63ea6 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -529,7 +529,7 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib SSdb* pSdb = pMnode->pSdb; SVgObj* pVgroup = NULL; SQueryPlan* pPlan = NULL; - SSubplan* plan = NULL; + SSubplan* pSubplan = NULL; if (pTopic->subType == TOPIC_SUB_TYPE__COLUMN) { pPlan = qStringToQueryPlan(pTopic->physicalPlan); @@ -545,21 +545,25 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib return -1; } - SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0); + SNodeListNode* pNodeListNode = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0); - int32_t opNum = LIST_LENGTH(inner->pNodeList); + int32_t opNum = LIST_LENGTH(pNodeListNode->pNodeList); if (opNum != 1) { qDestroyQueryPlan(pPlan); terrno = TSDB_CODE_MND_INVALID_TOPIC_QUERY; return -1; } - plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0); + + pSubplan = (SSubplan*)nodesListGetNode(pNodeListNode->pNodeList, 0); } void* pIter = NULL; while (1) { pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); - if (pIter == NULL) break; + if (pIter == NULL) { + break; + } + if (!mndVgroupInDb(pVgroup, pTopic->dbUid)) { sdbRelease(pSdb, pVgroup); continue; @@ -572,15 +576,15 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib pVgEp->vgId = pVgroup->vgId; taosArrayPush(pSub->unassignedVgs, &pVgEp); - mDebug("init subscription %s, assign vg: %d", pSub->key, pVgEp->vgId); + mDebug("init subscription %s for topic:%s assign vgId:%d", pSub->key, pTopic->name, pVgEp->vgId); if (pTopic->subType == TOPIC_SUB_TYPE__COLUMN) { int32_t msgLen; - plan->execNode.epSet = pVgEp->epSet; - plan->execNode.nodeId = pVgEp->vgId; + pSubplan->execNode.epSet = pVgEp->epSet; + pSubplan->execNode.nodeId = pVgEp->vgId; - if (qSubPlanToString(plan, &pVgEp->qmsg, &msgLen) < 0) { + if (qSubPlanToString(pSubplan, &pVgEp->qmsg, &msgLen) < 0) { sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); terrno = TSDB_CODE_QRY_INVALID_INPUT; @@ -594,6 +598,5 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib } qDestroyQueryPlan(pPlan); - return 0; } diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index c10851cfad..7b2005ef00 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -1256,19 +1256,19 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc STR_TO_VARSTR(n3, (char *)tNameGetTableName(&stbName)); SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)n1, false); + colDataSetVal(pColInfo, numOfRows, (const char *)n1, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)n2, false); + colDataSetVal(pColInfo, numOfRows, (const char *)n2, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)n3, false); + colDataSetVal(pColInfo, numOfRows, (const char *)n3, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pSma->dstVgId, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pSma->dstVgId, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pSma->createdTime, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pSma->createdTime, false); char col[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(col, (char *)""); diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index e6a253bcc0..5e98380a08 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -441,16 +441,16 @@ static int32_t mndRetrieveSnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB cols = 0; SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pObj->id, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pObj->id, false); char ep[TSDB_EP_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(ep, pObj->pDnode->ep, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)ep, false); + colDataSetVal(pColInfo, numOfRows, (const char *)ep, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pObj->createdTime, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pObj->createdTime, false); numOfRows++; sdbRelease(pSdb, pObj); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 499136084e..780d0461c8 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -2743,7 +2743,7 @@ void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t // STR_TO_VARSTR(comment, ""); // colDataAppend(pColInfo, numOfRows, comment, false); // } else { -// colDataAppendNULL(pColInfo, numOfRows); +// colDataSetNULL(pColInfo, numOfRows); // } // // char watermark[64 + VARSTR_HEADER_SIZE] = {0}; @@ -2932,7 +2932,7 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc varDataSetLen(stbName, strlen(&stbName[VARSTR_HEADER_SIZE])); SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)stbName, false); + colDataSetVal(pColInfo, numOfRows, (const char *)stbName, false); char db[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; tNameFromString(&name, pStb->db, T_NAME_ACCT | T_NAME_DB); @@ -2940,31 +2940,31 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc varDataSetLen(db, strlen(varDataVal(db))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)db, false); + colDataSetVal(pColInfo, numOfRows, (const char *)db, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pStb->createdTime, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->createdTime, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pStb->numOfColumns, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->numOfColumns, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pStb->numOfTags, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->numOfTags, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pStb->updateTime, false); // number of tables + colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->updateTime, false); // number of tables pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); if (pStb->commentLen > 0) { char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(comment, pStb->comment); - colDataAppend(pColInfo, numOfRows, comment, false); + colDataSetVal(pColInfo, numOfRows, comment, false); } else if (pStb->commentLen == 0) { char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(comment, ""); - colDataAppend(pColInfo, numOfRows, comment, false); + colDataSetVal(pColInfo, numOfRows, comment, false); } else { - colDataAppendNULL(pColInfo, numOfRows); + colDataSetNULL(pColInfo, numOfRows); } char watermark[64 + VARSTR_HEADER_SIZE] = {0}; @@ -2972,14 +2972,14 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc varDataSetLen(watermark, strlen(varDataVal(watermark))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)watermark, false); + colDataSetVal(pColInfo, numOfRows, (const char *)watermark, false); char maxDelay[64 + VARSTR_HEADER_SIZE] = {0}; sprintf(varDataVal(maxDelay), "%" PRId64 "a,%" PRId64 "a", pStb->maxdelay[0], pStb->maxdelay[1]); varDataSetLen(maxDelay, strlen(varDataVal(maxDelay))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)maxDelay, false); + colDataSetVal(pColInfo, numOfRows, (const char *)maxDelay, false); char rollup[160 + VARSTR_HEADER_SIZE] = {0}; int32_t rollupNum = (int32_t)taosArrayGetSize(pStb->pFuncs); @@ -2998,7 +2998,7 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc varDataSetLen(rollup, strlen(varDataVal(rollup))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)rollup, false); + colDataSetVal(pColInfo, numOfRows, (const char *)rollup, false); numOfRows++; sdbRelease(pSdb, pStb); @@ -3070,7 +3070,7 @@ static int32_t buildDbColsInfoBlock(const SSDataBlock *p, const SSysTableMeta *p colDataAppend(pColInfoData, numOfRows, (const char *)&pm->schema[j].bytes, false); for (int32_t k = 6; k <= 8; ++k) { pColInfoData = taosArrayGet(p->pDataBlock, k); - colDataAppendNULL(pColInfoData, numOfRows); + colDataSetNULL(pColInfoData, numOfRows); } numOfRows += 1; @@ -3174,7 +3174,7 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB colDataAppend(pColInfo, numOfRows, (const char *)&pStb->pColumns[i].bytes, false); while (cols < pShow->numOfColumns) { pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppendNULL(pColInfo, numOfRows); + colDataSetNULL(pColInfo, numOfRows); } numOfRows++; } diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 47ebdd706d..089a93bea3 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -507,7 +507,8 @@ static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStre SMCreateStbReq createReq = {0}; tstrncpy(createReq.name, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN); createReq.numOfColumns = pStream->outputSchema.nCols; - createReq.pColumns = taosArrayInit_s(createReq.numOfColumns, sizeof(SField), createReq.numOfColumns); + createReq.numOfTags = 1; // group id + createReq.pColumns = taosArrayInit_s(sizeof(SField), createReq.numOfColumns); // build fields for (int32_t i = 0; i < createReq.numOfColumns; i++) { SField *pField = taosArrayGet(createReq.pColumns, i); @@ -519,7 +520,7 @@ static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStre if (pStream->tagSchema.nCols == 0) { createReq.numOfTags = 1; - createReq.pTags = taosArrayInit_s(createReq.numOfTags, sizeof(SField), 1); + createReq.pTags = taosArrayInit_s(sizeof(SField), 1); // build tags SField *pField = taosArrayGet(createReq.pTags, 0); strcpy(pField->name, "group_id"); @@ -528,7 +529,7 @@ static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStre pField->bytes = 8; } else { createReq.numOfTags = pStream->tagSchema.nCols; - createReq.pTags = taosArrayInit_s(createReq.numOfTags, sizeof(SField), createReq.numOfTags); + createReq.pTags = taosArrayInit_s(sizeof(SField), createReq.numOfTags); for (int32_t i = 0; i < createReq.numOfTags; i++) { SField *pField = taosArrayGet(createReq.pTags, i); pField->bytes = pStream->tagSchema.pSchema[i].bytes; @@ -1104,52 +1105,52 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB char streamName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(streamName, mndGetDbStr(pStream->name), sizeof(streamName)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)streamName, false); + colDataSetVal(pColInfo, numOfRows, (const char *)streamName, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pStream->createTime, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pStream->createTime, false); char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(sql, pStream->sql, sizeof(sql)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)sql, false); + colDataSetVal(pColInfo, numOfRows, (const char *)sql, false); char status[20 + VARSTR_HEADER_SIZE] = {0}; char status2[20] = {0}; mndShowStreamStatus(status2, pStream); STR_WITH_MAXSIZE_TO_VARSTR(status, status2, sizeof(status)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&status, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&status, false); char sourceDB[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(sourceDB, mndGetDbStr(pStream->sourceDb), sizeof(sourceDB)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&sourceDB, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&sourceDB, false); char targetDB[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(targetDB, mndGetDbStr(pStream->targetDb), sizeof(targetDB)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&targetDB, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&targetDB, false); if (pStream->targetSTbName[0] == 0) { pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, NULL, true); + colDataSetVal(pColInfo, numOfRows, NULL, true); } else { char targetSTB[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(targetSTB, mndGetStbStr(pStream->targetSTbName), sizeof(targetSTB)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&targetSTB, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&targetSTB, false); } pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pStream->watermark, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pStream->watermark, false); char trigger[20 + VARSTR_HEADER_SIZE] = {0}; char trigger2[20] = {0}; mndShowStreamTrigger(trigger2, pStream); STR_WITH_MAXSIZE_TO_VARSTR(trigger, trigger2, sizeof(trigger)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&trigger, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&trigger, false); numOfRows++; sdbRelease(pSdb, pStream); @@ -1201,11 +1202,11 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock char streamName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(streamName, mndGetDbStr(pStream->name), sizeof(streamName)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)streamName, false); + colDataSetVal(pColInfo, numOfRows, (const char *)streamName, false); // task id pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pTask->taskId, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pTask->taskId, false); // node type char nodeType[20 + VARSTR_HEADER_SIZE] = {0}; @@ -1216,12 +1217,12 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock } else { memcpy(varDataVal(nodeType), "snode", 5); } - colDataAppend(pColInfo, numOfRows, nodeType, false); + colDataSetVal(pColInfo, numOfRows, nodeType, false); // node id pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); int32_t nodeId = TMAX(pTask->nodeId, 0); - colDataAppend(pColInfo, numOfRows, (const char *)&nodeId, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&nodeId, false); // level char level[20 + VARSTR_HEADER_SIZE] = {0}; @@ -1237,7 +1238,7 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock } else if (pTask->taskLevel == TASK_LEVEL__SINK) { } pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&level, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&level, false); // status char status[20 + VARSTR_HEADER_SIZE] = {0}; @@ -1245,7 +1246,7 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock strcpy(status, "normal"); STR_WITH_MAXSIZE_TO_VARSTR(status, status2, sizeof(status)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&status, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&status, false); numOfRows++; } diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index d127ceacf5..4bf8f42506 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -39,12 +39,10 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw); static int32_t mndSubActionInsert(SSdb *pSdb, SMqSubscribeObj *); static int32_t mndSubActionDelete(SSdb *pSdb, SMqSubscribeObj *); static int32_t mndSubActionUpdate(SSdb *pSdb, SMqSubscribeObj *pOldSub, SMqSubscribeObj *pNewSub); - -static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg); -static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg); - -static int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); -static void mndCancelGetNextSubscribe(SMnode *pMnode, void *pIter); +static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg); +static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg); +static int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); +static void mndCancelGetNextSubscribe(SMnode *pMnode, void *pIter); static int32_t mndSetSubRedoLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub) { SSdbRaw *pRedoRaw = mndSubActionEncode(pSub); @@ -85,12 +83,13 @@ int32_t mndInitSubscribe(SMnode *pMnode) { return sdbSetTable(pMnode->pSdb, table); } -static SMqSubscribeObj *mndCreateSub(SMnode *pMnode, const SMqTopicObj *pTopic, const char *subKey) { +static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj *pTopic, const char *subKey) { SMqSubscribeObj *pSub = tNewSubscribeObj(subKey); if (pSub == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } + pSub->dbUid = pTopic->dbUid; pSub->stbUid = pTopic->stbUid; pSub->subType = pTopic->subType; @@ -205,7 +204,7 @@ static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) { int32_t totalVgNum = pOutput->pSub->vgNum; const char *sub = pOutput->pSub->key; - mInfo("sub:%s, mq rebalance vgNum:%d", sub, pOutput->pSub->vgNum); + mInfo("sub:%s mq re-balance %d vgroups", sub, pOutput->pSub->vgNum); // 1. build temporary hash(vgId -> SMqRebOutputVg) to store modified vg SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); @@ -214,7 +213,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR int32_t removedNum = taosArrayGetSize(pInput->pRebInfo->removedConsumers); int32_t actualRemoved = 0; for (int32_t i = 0; i < removedNum; i++) { - int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->removedConsumers, i); + uint64_t consumerId = *(uint64_t *)taosArrayGet(pInput->pRebInfo->removedConsumers, i); SMqConsumerEp *pConsumerEp = taosHashGet(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t)); @@ -229,7 +228,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); - mInfo("sub:%s, mq rebalance remove vgId:%d from consumer:%" PRId64, sub, pVgEp->vgId, consumerId); + mInfo("sub:%s mq re-balance remove vgId:%d from consumer:%" PRId64, sub, pVgEp->vgId, consumerId); } taosArrayDestroy(pConsumerEp->vgs); taosHashRemove(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t)); @@ -239,7 +238,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR } if (removedNum != actualRemoved) { - mError("sub:%s, mq rebalance removedNum:%d not matched with actual:%d", sub, removedNum, actualRemoved); + mError("sub:%s mq re-balance removedNum:%d not matched with actual:%d", sub, removedNum, actualRemoved); } // if previously no consumer, there are vgs not assigned @@ -253,7 +252,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &rebOutput, sizeof(SMqRebOutputVg)); - mInfo("sub:%s, mq rebalance remove vgId:%d from unassigned", sub, pVgEp->vgId); + mInfo("sub:%s mq re-balance remove vgId:%d from unassigned", sub, pVgEp->vgId); } } @@ -267,7 +266,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR minVgCnt = totalVgNum / afterRebConsumerNum; imbConsumerNum = totalVgNum % afterRebConsumerNum; } - mInfo("sub:%s, mq rebalance %d consumer after rebalance, at least %d vg each, %d consumer has more vg", sub, + + mInfo("sub:%s mq re-balance %d consumers: at least %d vg each, %d consumer has more vg", sub, afterRebConsumerNum, minVgCnt, imbConsumerNum); // 4. first scan: remove consumer more than wanted, put to remove hash @@ -275,7 +275,10 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR void *pIter = NULL; while (1) { pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); - if (pIter == NULL) break; + if (pIter == NULL) { + break; + } + SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; int32_t consumerVgNum = taosArrayGetSize(pConsumerEp->vgs); @@ -297,7 +300,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); - mInfo("sub:%s, mq rebalance remove vgId:%d from consumer:%" PRId64 ",(first scan)", sub, pVgEp->vgId, + mInfo("sub:%s mq rebalance remove vgId:%d from consumer:%" PRId64 ",(first scan)", sub, pVgEp->vgId, pConsumerEp->consumerId); } imbCnt++; @@ -312,7 +315,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); - mInfo("sub:%s, mq rebalance remove vgId:%d from consumer:%" PRId64 ",(first scan)", sub, pVgEp->vgId, + mInfo("sub:%s mq rebalance remove vgId:%d from consumer:%" PRId64 ",(first scan)", sub, pVgEp->vgId, pConsumerEp->consumerId); } } @@ -330,7 +333,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR newConsumerEp.vgs = taosArrayInit(0, sizeof(void *)); taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, sizeof(SMqConsumerEp)); taosArrayPush(pOutput->newConsumers, &consumerId); - mInfo("sub:%s, mq rebalance add new consumer:%" PRId64, sub, consumerId); + mInfo("sub:%s mq rebalance add new consumer:%" PRId64, sub, consumerId); } } @@ -349,7 +352,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR // iter hash and find one vg pRemovedIter = taosHashIterate(pHash, pRemovedIter); if (pRemovedIter == NULL) { - mError("sub:%s, removed iter is null", sub); + mError("sub:%s removed iter is null", sub); continue; } @@ -402,33 +405,36 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR SMqRebOutputVg *pRebOutput = NULL; while (1) { pIter = taosHashIterate(pHash, pIter); - if (pIter == NULL) break; + if (pIter == NULL) { + break; + } + pRebOutput = (SMqRebOutputVg *)pIter; taosArrayPush(pOutput->pSub->unassignedVgs, &pRebOutput->pVgEp); taosArrayPush(pOutput->rebVgs, pRebOutput); - mInfo("sub:%s, mq rebalance unassign vgId:%d (second scan)", sub, pRebOutput->pVgEp->vgId); + mInfo("sub:%s mq re-balance unassign vgId:%d (second scan)", sub, pRebOutput->pVgEp->vgId); } } // 8. generate logs - mInfo("sub:%s, mq rebalance calculation completed, rebalanced vg", sub); + mInfo("sub:%s mq re-balance calculation completed, re-balanced vg", sub); for (int32_t i = 0; i < taosArrayGetSize(pOutput->rebVgs); i++) { SMqRebOutputVg *pOutputRebVg = taosArrayGet(pOutput->rebVgs, i); - mInfo("sub:%s, mq rebalance vgId:%d, moved from consumer:%" PRId64 ", to consumer:%" PRId64, sub, + mInfo("sub:%s mq re-balance vgId:%d, moved from consumer:0x%" PRIx64 ", to consumer:0x%" PRIx64, sub, pOutputRebVg->pVgEp->vgId, pOutputRebVg->oldConsumerId, pOutputRebVg->newConsumerId); } { - void *pIter = NULL; + pIter = NULL; while (1) { pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); if (pIter == NULL) break; SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; int32_t sz = taosArrayGetSize(pConsumerEp->vgs); - mInfo("sub:%s, mq rebalance final cfg: consumer %" PRId64 " has %d vg", sub, pConsumerEp->consumerId, sz); + mInfo("sub:%s mq re-balance final cfg: consumer:0x%" PRId64 " has %d vg", sub, pConsumerEp->consumerId, sz); for (int32_t i = 0; i < sz; i++) { SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, i); - mInfo("sub:%s, mq rebalance final cfg: vg %d to consumer %" PRId64 "", sub, pVgEp->vgId, + mInfo("sub:%s mq re-balance final cfg: vg %d to consumer:0x%" PRId64, sub, pVgEp->vgId, pConsumerEp->consumerId); } } @@ -552,11 +558,14 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { SMqDoRebalanceMsg *pReq = pMsg->pCont; void *pIter = NULL; - mInfo("mq rebalance start"); + mInfo("mq re-balance start"); while (1) { pIter = taosHashIterate(pReq->rebSubHash, pIter); - if (pIter == NULL) break; + if (pIter == NULL) { + break; + } + SMqRebInputObj rebInput = {0}; SMqRebOutputObj rebOutput = {0}; @@ -577,12 +586,13 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { mndSplitSubscribeKey(pRebInfo->key, topic, cgroup, true); SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); if (pTopic == NULL) { - mError("mq rebalance %s failed since topic %s not exist, abort", pRebInfo->key, topic); + mError("mq re-balance %s ignored since topic %s not exist", pRebInfo->key, topic); continue; } + taosRLockLatch(&pTopic->lock); - rebOutput.pSub = mndCreateSub(pMnode, pTopic, pRebInfo->key); + rebOutput.pSub = mndCreateSubscription(pMnode, pTopic, pRebInfo->key); if (rebOutput.pSub == NULL) { mError("mq rebalance %s failed create sub since %s, abort", pRebInfo->key, terrstr()); @@ -605,15 +615,16 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { } if (mndDoRebalance(pMnode, &rebInput, &rebOutput) < 0) { - mError("mq rebalance internal error"); + mError("mq re-balance internal error"); } // if add more consumer to balanced subscribe, // possibly no vg is changed if (mndPersistRebResult(pMnode, pMsg, &rebOutput) < 0) { - mError("mq rebalance persist rebalance output error, possibly vnode splitted or dropped"); + mError("mq re-balance persist re-balance output error, possibly vnode splitted or dropped"); } + taosArrayDestroy(pRebInfo->lostConsumers); taosArrayDestroy(pRebInfo->newConsumers); taosArrayDestroy(pRebInfo->removedConsumers); @@ -627,19 +638,18 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { } // reset flag - mInfo("mq rebalance completed successfully"); + mInfo("mq re-balance completed successfully"); taosHashCleanup(pReq->rebSubHash); mndRebEnd(); return 0; } -static int32_t mndProcessDropCgroupReq(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; - SSdb *pSdb = pMnode->pSdb; +static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) { + SMnode *pMnode = pMsg->info.node; SMDropCgroupReq dropReq = {0}; - if (tDeserializeSMDropCgroupReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { + if (tDeserializeSMDropCgroupReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; return -1; } @@ -663,7 +673,7 @@ static int32_t mndProcessDropCgroupReq(SRpcMsg *pReq) { return -1; } - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "drop-cgroup"); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pMsg, "drop-cgroup"); if (pTrans == NULL) { mError("cgroup: %s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, terrstr()); mndReleaseSubscribe(pMnode, pSub); @@ -956,7 +966,7 @@ END: return code; } -static int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity) { +int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; @@ -998,18 +1008,18 @@ static int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock varDataSetLen(cgroup, strlen(varDataVal(cgroup))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)topic, false); + colDataSetVal(pColInfo, numOfRows, (const char *)topic, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)cgroup, false); + colDataSetVal(pColInfo, numOfRows, (const char *)cgroup, false); // vg id pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pVgEp->vgId, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pVgEp->vgId, false); // consumer id pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pConsumerEp->consumerId, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumerEp->consumerId, false); mDebug("mnd show subscriptions: topic %s, consumer %" PRId64 " cgroup %s vgid %d", varDataVal(topic), pConsumerEp->consumerId, varDataVal(cgroup), pVgEp->vgId); @@ -1018,11 +1028,11 @@ static int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock #if 0 // subscribe time pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pSub->subscribeTime, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pSub->subscribeTime, false); // rebalance time pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pSub->rebalanceTime, pConsumer->rebalanceTime == 0); + colDataSetVal(pColInfo, numOfRows, (const char *)&pSub->rebalanceTime, pConsumer->rebalanceTime == 0); #endif numOfRows++; @@ -1046,18 +1056,18 @@ static int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock varDataSetLen(cgroup, strlen(varDataVal(cgroup))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)topic, false); + colDataSetVal(pColInfo, numOfRows, (const char *)topic, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)cgroup, false); + colDataSetVal(pColInfo, numOfRows, (const char *)cgroup, false); // vg id pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pVgEp->vgId, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pVgEp->vgId, false); // consumer id pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, NULL, true); + colDataSetVal(pColInfo, numOfRows, NULL, true); mDebug("mnd show subscriptions(unassigned): topic %s, cgroup %s vgid %d", varDataVal(topic), varDataVal(cgroup), pVgEp->vgId); @@ -1066,11 +1076,11 @@ static int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock #if 0 // subscribe time pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pSub->subscribeTime, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pSub->subscribeTime, false); // rebalance time pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pSub->rebalanceTime, pConsumer->rebalanceTime == 0); + colDataSetVal(pColInfo, numOfRows, (const char *)&pSub->rebalanceTime, pConsumer->rebalanceTime == 0); #endif numOfRows++; @@ -1090,7 +1100,7 @@ static int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock return numOfRows; } -static void mndCancelGetNextSubscribe(SMnode *pMnode, void *pIter) { +void mndCancelGetNextSubscribe(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); } diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 96dba24566..dd03599c2b 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -33,7 +33,7 @@ static int32_t mndTopicActionInsert(SSdb *pSdb, SMqTopicObj *pTopic); static int32_t mndTopicActionDelete(SSdb *pSdb, SMqTopicObj *pTopic); -static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pTopic, SMqTopicObj *pNewTopic); +static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pOldTopic, SMqTopicObj *pNewTopic); static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq); static int32_t mndProcessDropTopicReq(SRpcMsg *pReq); @@ -561,17 +561,17 @@ static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) { goto _OVER; } - mInfo("topic:%s, start to create, sql:%s", createTopicReq.name, createTopicReq.sql); + mInfo("topic:%s start to create, sql:%s", createTopicReq.name, createTopicReq.sql); if (mndCheckCreateTopicReq(&createTopicReq) != 0) { - mError("topic:%s, failed to create since %s", createTopicReq.name, terrstr()); + mError("topic:%s failed to create since %s", createTopicReq.name, terrstr()); goto _OVER; } pTopic = mndAcquireTopic(pMnode, createTopicReq.name); if (pTopic != NULL) { if (createTopicReq.igExists) { - mInfo("topic:%s, already exist, ignore exist is set", createTopicReq.name); + mInfo("topic:%s already exist, ignore exist is set", createTopicReq.name); code = 0; goto _OVER; } else { @@ -726,8 +726,9 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { if (pTopic->ntbUid != 0) { // broadcast to all vnode - void *pIter = NULL; + pIter = NULL; SVgObj *pVgroup = NULL; + while (1) { pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); if (pIter == NULL) break; @@ -813,23 +814,23 @@ static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl /*tNameGetDbName(&n, varDataVal(topicName));*/ varDataSetLen(topicName, strlen(varDataVal(topicName))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)topicName, false); + colDataSetVal(pColInfo, numOfRows, (const char *)topicName, false); char dbName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; tNameFromString(&n, pTopic->db, T_NAME_ACCT | T_NAME_DB); tNameGetDbName(&n, varDataVal(dbName)); varDataSetLen(dbName, strlen(varDataVal(dbName))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)dbName, false); + colDataSetVal(pColInfo, numOfRows, (const char *)dbName, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pTopic->createTime, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pTopic->createTime, false); char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0}; tstrncpy(&sql[VARSTR_HEADER_SIZE], pTopic->sql, TSDB_SHOW_SQL_LEN); varDataSetLen(sql, strlen(&sql[VARSTR_HEADER_SIZE])); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)sql, false); + colDataSetVal(pColInfo, numOfRows, (const char *)sql, false); numOfRows++; sdbRelease(pSdb, pTopic); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 5776d8a559..cfe7b643a9 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1664,36 +1664,36 @@ static int32_t mndRetrieveTrans(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl cols = 0; SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pTrans->id, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pTrans->id, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pTrans->createdTime, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pTrans->createdTime, false); char stage[TSDB_TRANS_STAGE_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(stage, mndTransStr(pTrans->stage), pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)stage, false); + colDataSetVal(pColInfo, numOfRows, (const char *)stage, false); char opername[TSDB_TRANS_OPER_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(opername, pTrans->opername, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)opername, false); + colDataSetVal(pColInfo, numOfRows, (const char *)opername, false); char dbname[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(dbname, mndGetDbStr(pTrans->dbname), pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)dbname, false); + colDataSetVal(pColInfo, numOfRows, (const char *)dbname, false); char stbname[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(stbname, mndGetDbStr(pTrans->stbname), pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)stbname, false); + colDataSetVal(pColInfo, numOfRows, (const char *)stbname, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pTrans->failedTimes, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pTrans->failedTimes, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pTrans->lastExecTime, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pTrans->lastExecTime, false); char lastInfo[TSDB_TRANS_ERROR_LEN + VARSTR_HEADER_SIZE] = {0}; char detail[TSDB_TRANS_ERROR_LEN + 1] = {0}; @@ -1709,7 +1709,7 @@ static int32_t mndRetrieveTrans(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl } STR_WITH_MAXSIZE_TO_VARSTR(lastInfo, detail, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)lastInfo, false); + colDataSetVal(pColInfo, numOfRows, (const char *)lastInfo, false); numOfRows++; sdbRelease(pSdb, pTrans); diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index b965e13316..a03d640ad5 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -795,23 +795,23 @@ static int32_t mndRetrieveUsers(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols); char name[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(name, pUser->user, pShow->pMeta->pSchemas[cols].bytes); - colDataAppend(pColInfo, numOfRows, (const char *)name, false); + colDataSetVal(pColInfo, numOfRows, (const char *)name, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); - colDataAppend(pColInfo, numOfRows, (const char *)&pUser->superUser, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pUser->superUser, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); - colDataAppend(pColInfo, numOfRows, (const char *)&pUser->enable, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pUser->enable, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); - colDataAppend(pColInfo, numOfRows, (const char *)&pUser->sysInfo, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pUser->sysInfo, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); - colDataAppend(pColInfo, numOfRows, (const char *)&pUser->createdTime, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pUser->createdTime, false); numOfRows++; sdbRelease(pSdb, pUser); @@ -848,17 +848,17 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(userName, pUser->user, pShow->pMeta->pSchemas[cols].bytes); SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)userName, false); + colDataSetVal(pColInfo, numOfRows, (const char *)userName, false); char privilege[20] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(privilege, "all", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)privilege, false); + colDataSetVal(pColInfo, numOfRows, (const char *)privilege, false); char objName[20] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(objName, "all", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)objName, false); + colDataSetVal(pColInfo, numOfRows, (const char *)objName, false); numOfRows++; } @@ -869,12 +869,12 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(userName, pUser->user, pShow->pMeta->pSchemas[cols].bytes); SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)userName, false); + colDataSetVal(pColInfo, numOfRows, (const char *)userName, false); char privilege[20] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(privilege, "read", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)privilege, false); + colDataSetVal(pColInfo, numOfRows, (const char *)privilege, false); SName name = {0}; char objName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -882,7 +882,7 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock tNameGetDbName(&name, varDataVal(objName)); varDataSetLen(objName, strlen(varDataVal(objName))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)objName, false); + colDataSetVal(pColInfo, numOfRows, (const char *)objName, false); numOfRows++; db = taosHashIterate(pUser->readDbs, db); @@ -894,12 +894,12 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(userName, pUser->user, pShow->pMeta->pSchemas[cols].bytes); SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)userName, false); + colDataSetVal(pColInfo, numOfRows, (const char *)userName, false); char privilege[20] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(privilege, "write", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)privilege, false); + colDataSetVal(pColInfo, numOfRows, (const char *)privilege, false); SName name = {0}; char objName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -907,7 +907,7 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock tNameGetDbName(&name, varDataVal(objName)); varDataSetLen(objName, strlen(varDataVal(objName))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)objName, false); + colDataSetVal(pColInfo, numOfRows, (const char *)objName, false); numOfRows++; db = taosHashIterate(pUser->writeDbs, db); @@ -919,18 +919,18 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(userName, pUser->user, pShow->pMeta->pSchemas[cols].bytes); SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)userName, false); + colDataSetVal(pColInfo, numOfRows, (const char *)userName, false); char privilege[20] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(privilege, "subscribe", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)privilege, false); + colDataSetVal(pColInfo, numOfRows, (const char *)privilege, false); char topicName[TSDB_TOPIC_NAME_LEN + VARSTR_HEADER_SIZE + 5] = {0}; tstrncpy(varDataVal(topicName), mndGetDbStr(topic), TSDB_TOPIC_NAME_LEN - 2); varDataSetLen(topicName, strlen(varDataVal(topicName))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)topicName, false); + colDataSetVal(pColInfo, numOfRows, (const char *)topicName, false); numOfRows++; topic = taosHashIterate(pUser->topics, topic); diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 171916b637..b7bcaf41fd 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -529,10 +529,10 @@ SArray *mndBuildDnodesArray(SMnode *pMnode, int32_t exceptDnodeId) { } static int32_t mndCompareDnodeId(int32_t *dnode1Id, int32_t *dnode2Id) { - if (*dnode1Id == *dnode2Id) { - return 0; - } - return *dnode1Id > *dnode2Id ? 1 : -1; + if (*dnode1Id == *dnode2Id) { + return 0; + } + return *dnode1Id > *dnode2Id ? 1 : -1; } static float mndGetDnodeScore(SDnodeObj *pDnode, int32_t additionDnodes, float ratio) { @@ -541,12 +541,12 @@ static float mndGetDnodeScore(SDnodeObj *pDnode, int32_t additionDnodes, float r } static int32_t mndCompareDnodeVnodes(SDnodeObj *pDnode1, SDnodeObj *pDnode2) { - float d1Score = mndGetDnodeScore(pDnode1, 0, 0.9); - float d2Score = mndGetDnodeScore(pDnode2, 0, 0.9); - if (d1Score == d2Score) { - return 0; - } - return d1Score > d2Score ? 1 : -1; + float d1Score = mndGetDnodeScore(pDnode1, 0, 0.9); + float d2Score = mndGetDnodeScore(pDnode2, 0, 0.9); + if (d1Score == d2Score) { + return 0; + } + return d1Score > d2Score ? 1 : -1; } void mndSortVnodeGid(SVgObj *pVgroup) { @@ -739,7 +739,7 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p cols = 0; SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->vgId, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pVgroup->vgId, false); SName name = {0}; char db[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -748,17 +748,17 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p varDataSetLen(db, strlen(varDataVal(db))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)db, false); + colDataSetVal(pColInfo, numOfRows, (const char *)db, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->numOfTables, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pVgroup->numOfTables, false); // default 3 replica, add 1 replica if move vnode for (int32_t i = 0; i < 4; ++i) { pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); if (i < pVgroup->replica) { int16_t dnodeId = (int16_t)pVgroup->vnodeGid[i].dnodeId; - colDataAppend(pColInfo, numOfRows, (const char *)&dnodeId, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&dnodeId, false); bool exist = false; bool online = false; @@ -789,24 +789,24 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p STR_WITH_MAXSIZE_TO_VARSTR(buf1, role, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)buf1, false); + colDataSetVal(pColInfo, numOfRows, (const char *)buf1, false); } else { - colDataAppendNULL(pColInfo, numOfRows); + colDataSetNULL(pColInfo, numOfRows); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppendNULL(pColInfo, numOfRows); + colDataSetNULL(pColInfo, numOfRows); } } pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); int32_t cacheUsage = (int32_t)pVgroup->cacheUsage; - colDataAppend(pColInfo, numOfRows, (const char *)&cacheUsage, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&cacheUsage, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->isTsma, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pVgroup->isTsma, false); // pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); // if (pDb == NULL || pDb->compactStartTime <= 0) { - // colDataAppendNULL(pColInfo, numOfRows); + // colDataSetNULL(pColInfo, numOfRows); // } else { // colDataAppend(pColInfo, numOfRows, (const char *)&pDb->compactStartTime, false); // } @@ -905,15 +905,15 @@ static int32_t mndRetrieveVnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB cols = 0; pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->vgId, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pVgroup->vgId, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->replica, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pVgroup->replica, false); char buf[20] = {0}; STR_TO_VARSTR(buf, syncStr(pVgid->syncState)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)buf, false); + colDataSetVal(pColInfo, numOfRows, (const char *)buf, false); const char *dbname = mndGetDbStr(pVgroup->dbName); char b1[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -923,10 +923,10 @@ static int32_t mndRetrieveVnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB STR_WITH_MAXSIZE_TO_VARSTR(b1, "NULL", TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE); } pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)b1, false); + colDataSetVal(pColInfo, numOfRows, (const char *)b1, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pVgid->dnodeId, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pVgid->dnodeId, false); SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); char b2[TSDB_EP_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -936,7 +936,7 @@ static int32_t mndRetrieveVnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB STR_WITH_MAXSIZE_TO_VARSTR(b2, "NULL", TSDB_EP_LEN + VARSTR_HEADER_SIZE); } pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)b2, false); + colDataSetVal(pColInfo, numOfRows, (const char *)b2, false); numOfRows++; } diff --git a/source/dnode/qnode/inc/qndInt.h b/source/dnode/qnode/inc/qndInt.h index e8ccb75040..86deda52ad 100644 --- a/source/dnode/qnode/inc/qndInt.h +++ b/source/dnode/qnode/inc/qndInt.h @@ -29,7 +29,7 @@ extern "C" { #endif -typedef struct SQWorker SQHandle; +typedef struct SQueueWorker SQHandle; typedef struct SQnode { int32_t qndId; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index eda47bf5de..93e611e870 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -58,7 +58,7 @@ typedef struct STQ STQ; typedef struct SVState SVState; typedef struct SVStatis SVStatis; typedef struct SVBufPool SVBufPool; -typedef struct SQWorker SQHandle; +typedef struct SQueueWorker SQHandle; typedef struct STsdbKeepCfg STsdbKeepCfg; typedef struct SMetaSnapReader SMetaSnapReader; typedef struct SMetaSnapWriter SMetaSnapWriter; diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index a980b653a9..02766c8076 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -291,10 +291,7 @@ static void tdDestroyRSmaStat(void *pRSmaStat) { // step 1: set rsma trigger stat cancelled atomic_store_8(RSMA_TRIGGER_STAT(pStat), TASK_TRIGGER_STAT_CANCELLED); - // step 2: destroy the rsma info and associated fetch tasks - taosHashCleanup(RSMA_INFO_HASH(pStat)); - - // step 3: wait for all triggered fetch tasks to finish + // step 2: wait for all triggered fetch tasks to finish int32_t nLoops = 0; while (1) { if (T_REF_VAL_GET((SSmaStat *)pStat) == 0) { @@ -310,9 +307,12 @@ static void tdDestroyRSmaStat(void *pRSmaStat) { } } - // step 4: + // step 3: tdRsmaStopExecutor(pSma); + // step 4: destroy the rsma info and associated fetch tasks + taosHashCleanup(RSMA_INFO_HASH(pStat)); + // step 5: tdRSmaFSClose(RSMA_FS(pStat)); diff --git a/source/dnode/vnode/src/sma/smaFS.c b/source/dnode/vnode/src/sma/smaFS.c index 5dbe91f836..1211ef9405 100644 --- a/source/dnode/vnode/src/sma/smaFS.c +++ b/source/dnode/vnode/src/sma/smaFS.c @@ -559,7 +559,7 @@ int32_t tdRSmaFSRef(SSma *pSma, SRSmaFS *pFS) { SRSmaFS *qFS = RSMA_FS(pStat); int32_t size = taosArrayGetSize(qFS->aQTaskInf); - pFS->aQTaskInf = taosArrayInit_s(size, sizeof(SQTaskFile), size); + pFS->aQTaskInf = taosArrayInit_s(sizeof(SQTaskFile), size); if (pFS->aQTaskInf == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _exit); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 9bdd8f4bdf..7eacf344fd 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1283,17 +1283,17 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { for (int32_t i = 0; i < sz; i++) { // start key column SColumnInfoData* pStartCol = taosArrayGet(pDelBlock->pDataBlock, START_TS_COLUMN_INDEX); - colDataAppend(pStartCol, i, (const char*)&pRes->skey, false); // end key column + colDataSetVal(pStartCol, i, (const char*)&pRes->skey, false); // end key column SColumnInfoData* pEndCol = taosArrayGet(pDelBlock->pDataBlock, END_TS_COLUMN_INDEX); - colDataAppend(pEndCol, i, (const char*)&pRes->ekey, false); + colDataSetVal(pEndCol, i, (const char*)&pRes->ekey, false); // uid column SColumnInfoData* pUidCol = taosArrayGet(pDelBlock->pDataBlock, UID_COLUMN_INDEX); int64_t* pUid = taosArrayGet(pRes->uidList, i); - colDataAppend(pUidCol, i, (const char*)pUid, false); + colDataSetVal(pUidCol, i, (const char*)pUid, false); - colDataAppendNULL(taosArrayGet(pDelBlock->pDataBlock, GROUPID_COLUMN_INDEX), i); - colDataAppendNULL(taosArrayGet(pDelBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX), i); - colDataAppendNULL(taosArrayGet(pDelBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX), i); + colDataSetNULL(taosArrayGet(pDelBlock->pDataBlock, GROUPID_COLUMN_INDEX), i); + colDataSetNULL(taosArrayGet(pDelBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX), i); + colDataSetNULL(taosArrayGet(pDelBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX), i); } taosArrayDestroy(pRes->uidList); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 414ffda544..a2fafdf7c3 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -693,7 +693,7 @@ int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbD goto FAIL; } } else { - colDataAppendNULL(pColData, i); + colDataSetNULL(pColData, i); } } else { if (colDataAppend(pColData, i, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) { @@ -734,7 +734,7 @@ int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbD goto FAIL; } } else { - colDataAppendNULL(pColData, i); + colDataSetNULL(pColData, i); } /*val = colVal.value.pData;*/ } else { @@ -861,7 +861,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader) { if (!tdSTSRowIterFetch(&iter, pColData->info.colId, pColData->info.type, &sVal)) { break; } - if (colDataAppend(pColData, curRow, sVal.val, sVal.valType != TD_VTYPE_NORM) < 0) { + if (colDataSetVal(pColData, curRow, sVal.val, sVal.valType != TD_VTYPE_NORM) < 0) { goto FAIL; } } @@ -986,7 +986,9 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas break; } - if (colDataAppend(pColData, curRow, sVal.val, sVal.valType == TD_VTYPE_NULL) < 0) { + ASSERT(sVal.valType != TD_VTYPE_NONE); + + if (colDataSetVal(pColData, curRow, sVal.val, sVal.valType == TD_VTYPE_NULL) < 0) { goto FAIL; } tqDebug("vgId:%d, row %d col %d append %d", pReader->pWalReader->pWal->cfg.vgId, curRow, i, @@ -1141,7 +1143,7 @@ int32_t tqRetrieveTaosxBlock2(STqReader* pReader, SArray* blocks, SArray* schema goto FAIL; } } else { - colDataAppendNULL(pColData, curRow - lastRow); + colDataSetNULL(pColData, curRow - lastRow); } } else { if (colDataAppend(pColData, curRow - lastRow, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) { @@ -1235,7 +1237,7 @@ int32_t tqRetrieveTaosxBlock2(STqReader* pReader, SArray* blocks, SArray* schema goto FAIL; } } else { - colDataAppendNULL(pColData, curRow - lastRow); + colDataSetNULL(pColData, curRow - lastRow); } } else { if (colDataAppend(pColData, curRow - lastRow, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) { @@ -1313,7 +1315,10 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { void* pIter = NULL; while (1) { pIter = taosHashIterate(pTq->pHandle, pIter); - if (pIter == NULL) break; + if (pIter == NULL) { + break; + } + STqHandle* pExec = (STqHandle*)pIter; if (pExec->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { int32_t code = qUpdateQualifiedTableId(pExec->execHandle.task, tbUidList, isAdd); diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index 6ea169a5e3..cf1abd40e1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -67,7 +67,7 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p // pColInfoData->info.bytes includes the VARSTR_HEADER_SIZE, need to substruct it p->hasResult = true; varDataSetLen(pRes[i], pColInfoData->info.bytes - VARSTR_HEADER_SIZE); - colDataAppend(pColInfoData, numOfRows, (const char*)pRes[i], false); + colDataSetVal(pColInfoData, numOfRows, (const char*)pRes[i], false); } pBlock->info.rows += allNullRow ? 0 : 1; @@ -77,7 +77,7 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p if (slotIds[i] == -1) { SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, 0); - colDataAppend(pColInfoData, numOfRows, (const char*)&pColVal->ts, false); + colDataSetVal(pColInfoData, numOfRows, (const char*)&pColVal->ts, false); } else { int32_t slotId = slotIds[i]; SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, slotId); @@ -85,14 +85,14 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p if (IS_VAR_DATA_TYPE(pColVal->colVal.type)) { if (!COL_VAL_IS_VALUE(&pColVal->colVal)) { - colDataAppendNULL(pColInfoData, numOfRows); + colDataSetNULL(pColInfoData, numOfRows); } else { varDataSetLen(pReader->transferBuf[slotId], pVal->value.nData); memcpy(varDataVal(pReader->transferBuf[slotId]), pVal->value.pData, pVal->value.nData); - colDataAppend(pColInfoData, numOfRows, pReader->transferBuf[slotId], false); + colDataSetVal(pColInfoData, numOfRows, pReader->transferBuf[slotId], false); } } else { - colDataAppend(pColInfoData, numOfRows, (const char*)&pVal->value.val, !COL_VAL_IS_VALUE(pVal)); + colDataSetVal(pColInfoData, numOfRows, (const char*)&pVal->value.val, !COL_VAL_IS_VALUE(pVal)); } } } diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 42728be657..9b3dbcd8ea 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -148,7 +148,7 @@ bool tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2) { return pDelFi int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype) { int32_t code = 0; - int64_t size; + int64_t size = 0; int64_t n; TdFilePtr pFD; char fname[TSDB_FILENAME_LEN]; @@ -167,7 +167,7 @@ int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype) { tPutSmaFile(hdr, pSet->pSmaF); break; default: - ASSERT(0); + goto _err; // make the coverity scan happy } taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index c83fdb2e4f..2345c61607 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -222,6 +222,8 @@ static bool hasDataInFileBlock(const SBlockData* pBlockData, const SFil static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter); static int32_t getInitialDelIndex(const SArray* pDelSkyline, int32_t order); +static STableBlockScanInfo* getTableBlockScanInfo(SHashObj* pTableMap, uint64_t uid, const char* id); + static FORCE_INLINE STSchema* getLatestTableSchema(STsdbReader* pReader, uint64_t uid); static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } @@ -488,7 +490,7 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdb return TSDB_CODE_SUCCESS; } -static int32_t filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader, bool *hasNext) { +static int32_t filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader, bool* hasNext) { bool asc = ASCENDING_TRAVERSE(pIter->order); int32_t step = asc ? 1 : -1; pIter->index += step; @@ -770,14 +772,11 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFReader* pFileReader, if (pBlockIdx->uid == pList->tableUidList[j]) { // this block belongs to a table that is not queried. - void* p = taosHashGet(pReader->status.pTableMap, &pBlockIdx->uid, sizeof(uint64_t)); - if (p == NULL) { - tsdbError("failed to locate the tableBlockScan Info in hashmap, uid:%" PRIu64 ", %s", pBlockIdx->uid, - pReader->idStr); - return TSDB_CODE_APP_ERROR; + STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, pBlockIdx->uid, pReader->idStr); + if (pScanInfo == NULL) { + return terrno; } - STableBlockScanInfo* pScanInfo = *(STableBlockScanInfo**)p; if (pScanInfo->pBlockList == NULL) { pScanInfo->pBlockList = taosArrayInit(4, sizeof(SBlockIndex)); } @@ -824,10 +823,11 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN cleanupTableScanInfo(pReader->status.pTableMap); for (int32_t i = 0; i < numOfTables; ++i) { - SBlockIdx* pBlockIdx = taosArrayGet(pIndexList, i); - - STableBlockScanInfo* pScanInfo = - *(STableBlockScanInfo**)taosHashGet(pReader->status.pTableMap, &pBlockIdx->uid, sizeof(int64_t)); + SBlockIdx* pBlockIdx = taosArrayGet(pIndexList, i); + STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, pBlockIdx->uid, pReader->idStr); + if (pScanInfo == NULL) { + return terrno; + } tMapDataReset(&pScanInfo->mapData); tsdbReadDataBlk(pReader->pFileReader, pBlockIdx, &pScanInfo->mapData); @@ -905,7 +905,7 @@ static void doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_ SBlockLoadSuppInfo* pSup) { if (IS_VAR_DATA_TYPE(pColVal->type)) { if (!COL_VAL_IS_VALUE(pColVal)) { - colDataAppendNULL(pColInfoData, rowIndex); + colDataSetNULL(pColInfoData, rowIndex); } else { varDataSetLen(pSup->buildBuf[colIndex], pColVal->value.nData); ASSERT(pColVal->value.nData <= pColInfoData->info.bytes); @@ -913,10 +913,10 @@ static void doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_ memcpy(varDataVal(pSup->buildBuf[colIndex]), pColVal->value.pData, pColVal->value.nData); } - colDataAppend(pColInfoData, rowIndex, pSup->buildBuf[colIndex], false); + colDataSetVal(pColInfoData, rowIndex, pSup->buildBuf[colIndex], false); } } else { - colDataAppend(pColInfoData, rowIndex, (const char*)&pColVal->value, !COL_VAL_IS_VALUE(pColVal)); + colDataSetVal(pColInfoData, rowIndex, (const char*)&pColVal->value, !COL_VAL_IS_VALUE(pColVal)); } } @@ -939,9 +939,7 @@ static int doBinarySearchKey(TSKEY* keyList, int num, int pos, TSKEY key, int or s = pos; // check - assert(pos >= 0 && pos < num); - assert(num > 0); - + ASSERT(pos >= 0 && pos < num && num > 0); if (order == TSDB_ORDER_ASC) { // find the first position which is smaller than the key e = num - 1; @@ -1191,7 +1189,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader) { pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); if (pData->flag == HAS_NONE || pData->flag == HAS_NULL || pData->flag == (HAS_NULL | HAS_NONE)) { - colDataAppendNNULL(pColData, 0, dumpedRows); + colDataSetNNULL(pColData, 0, dumpedRows); } else { if (IS_MATHABLE_TYPE(pColData->info.type)) { copyNumericCols(pData, pDumpInfo, pColData, dumpedRows, asc); @@ -1207,7 +1205,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader) { i += 1; } else { // the specified column does not exist in file block, fill with null data pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); - colDataAppendNNULL(pColData, 0, dumpedRows); + colDataSetNNULL(pColData, 0, dumpedRows); i += 1; } } @@ -1215,7 +1213,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader) { // fill the mis-matched columns with null value while (i < numOfOutputCols) { pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); - colDataAppendNNULL(pColData, 0, dumpedRows); + colDataSetNNULL(pColData, 0, dumpedRows); i += 1; } @@ -1342,14 +1340,13 @@ static int32_t fileDataBlockOrderCompar(const void* pLeft, const void* pRight, v static int32_t doSetCurrentBlock(SDataBlockIter* pBlockIter, const char* idStr) { SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); if (pBlockInfo != NULL) { - STableBlockScanInfo** pScanInfo = taosHashGet(pBlockIter->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); + STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pBlockIter->pTableMap, pBlockInfo->uid, idStr); if (pScanInfo == NULL) { - tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, %s", pBlockInfo->uid, idStr); - return TSDB_CODE_INVALID_PARA; + return terrno; } - SBlockIndex* pIndex = taosArrayGet((*pScanInfo)->pBlockList, pBlockInfo->tbBlockIdx); - tMapDataGetItemByIdx(&(*pScanInfo)->mapData, pIndex->ordinalIndex, &pBlockIter->block, tGetDataBlk); + SBlockIndex* pIndex = taosArrayGet(pScanInfo->pBlockList, pBlockInfo->tbBlockIdx); + tMapDataGetItemByIdx(&pScanInfo->mapData, pIndex->ordinalIndex, &pBlockIter->block, tGetDataBlk); } #if 0 @@ -2596,16 +2593,11 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { STableBlockScanInfo* pBlockScanInfo = NULL; if (pBlockInfo != NULL) { - void* p = taosHashGet(pReader->status.pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); - if (p == NULL) { - code = TSDB_CODE_INVALID_PARA; - tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", pBlockInfo->uid, - taosHashGetSize(pReader->status.pTableMap), pReader->idStr); + pBlockScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr); + if (pBlockScanInfo == NULL) { goto _end; } - pBlockScanInfo = *(STableBlockScanInfo**)p; - SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); TSDBKEY keyInBuf = getCurrentKeyInBuf(pBlockScanInfo, pReader); @@ -2802,13 +2794,13 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { SArray* pIndexList = taosArrayInit(numOfTables, sizeof(SBlockIdx)); while (1) { - bool hasNext = false; + bool hasNext = false; int32_t code = filesetIteratorNext(&pStatus->fileIter, pReader, &hasNext); if (code) { taosArrayDestroy(pIndexList); return code; } - + if (!hasNext) { // no data files on disk break; } @@ -2951,13 +2943,9 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; - ASSERT(pBlockInfo != NULL); - - pScanInfo = *(STableBlockScanInfo**)taosHashGet(pReader->status.pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); + pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr); if (pScanInfo == NULL) { - tsdbError("failed to get table scan-info, %s", pReader->idStr); - code = TSDB_CODE_INVALID_PARA; - return code; + return terrno; } pBlock = getCurrentBlock(pBlockIter); @@ -3776,7 +3764,7 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pT } else if (colId < pSchema->columns[j].colId) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); - colDataAppendNULL(pColInfoData, outputRowIndex); + colDataSetNULL(pColInfoData, outputRowIndex); i += 1; } else if (colId > pSchema->columns[j].colId) { j += 1; @@ -3786,7 +3774,7 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pT // set null value since current column does not exist in the "pSchema" while (i < pSupInfo->numOfCols) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); - colDataAppendNULL(pColInfoData, outputRowIndex); + colDataSetNULL(pColInfoData, outputRowIndex); i += 1; } @@ -3826,7 +3814,7 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S j += 1; } else if (pData->cid > pCol->info.colId) { // the specified column does not exist in file block, fill with null data - colDataAppendNULL(pCol, outputRowIndex); + colDataSetNULL(pCol, outputRowIndex); } i += 1; @@ -3834,7 +3822,7 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S while (i < numOfOutputCols) { SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); - colDataAppendNULL(pCol, outputRowIndex); + colDataSetNULL(pCol, outputRowIndex); i += 1; } @@ -4532,7 +4520,7 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, SSDataBlock* pResBlock = pReader->pResBlock; if (pResBlock->pBlockAgg == NULL) { size_t num = taosArrayGetSize(pResBlock->pDataBlock); - pResBlock->pBlockAgg = taosMemoryCalloc(num, sizeof(SColumnDataAgg)); + pResBlock->pBlockAgg = taosMemoryCalloc(num, POINTER_BYTES); } // do fill all null column value SMA info @@ -4562,15 +4550,23 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, return code; } +STableBlockScanInfo* getTableBlockScanInfo(SHashObj* pTableMap, uint64_t uid, const char* id) { + STableBlockScanInfo** p = taosHashGet(pTableMap, &uid, sizeof(uid)); + if (p == NULL || *p == NULL) { + terrno = TSDB_CODE_INVALID_PARA; + int32_t size = taosHashGetSize(pTableMap); + tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", uid, size, id); + return NULL; + } + + return *p; +} + static SSDataBlock* doRetrieveDataBlock(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pStatus->blockIter); - STableBlockScanInfo* pBlockScanInfo = - *(STableBlockScanInfo**)taosHashGet(pStatus->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); + STableBlockScanInfo* pBlockScanInfo = getTableBlockScanInfo(pStatus->pTableMap, pBlockInfo->uid, pReader->idStr); if (pBlockScanInfo == NULL) { - terrno = TSDB_CODE_INVALID_PARA; - tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", pBlockInfo->uid, - taosHashGetSize(pReader->status.pTableMap), pReader->idStr); return NULL; } diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index b075c3d835..0779388ba9 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -150,22 +150,21 @@ void vnodeUpdCommitSched(SVnode *pVnode) { } int vnodeShouldCommit(SVnode *pVnode) { - if (!pVnode->inUse || !osDataSpaceAvailable()) { - return false; - } - SVCommitSched *pSched = &pVnode->commitSched; int64_t nowMs = taosGetMonoTimestampMs(); + bool diskAvail = osDataSpaceAvailable(); + bool needCommit = false; - return (((pVnode->inUse->size > pVnode->inUse->node.size) && (pSched->commitMs + SYNC_VND_COMMIT_MIN_MS < nowMs)) || - (pVnode->inUse->size > 0 && pSched->commitMs + pSched->maxWaitMs < nowMs)); -} - -int vnodeShouldCommitOld(SVnode *pVnode) { - if (pVnode->inUse) { - return osDataSpaceAvailable() && (pVnode->inUse->size > pVnode->inUse->node.size); + taosThreadMutexLock(&pVnode->mutex); + if (!pVnode->inUse || !diskAvail) { + goto _out; } - return false; + needCommit = + (((pVnode->inUse->size > pVnode->inUse->node.size) && (pSched->commitMs + SYNC_VND_COMMIT_MIN_MS < nowMs)) || + (pVnode->inUse->size > 0 && pSched->commitMs + pSched->maxWaitMs < nowMs)); +_out: + taosThreadMutexUnlock(&pVnode->mutex); + return needCommit; } int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) { diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 89e92b0cc8..f62e93b82a 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -1707,7 +1707,7 @@ int32_t ctgLaunchGetTbMetasTask(SCtgTask* pTask) { return TSDB_CODE_SUCCESS; } - pTask->msgCtxs = taosArrayInit_s(pCtx->fetchNum, sizeof(SCtgMsgCtx), pCtx->fetchNum); + pTask->msgCtxs = taosArrayInit_s(sizeof(SCtgMsgCtx), pCtx->fetchNum); for (int32_t i = 0; i < pCtx->fetchNum; ++i) { SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); SName* pName = ctgGetFetchName(pCtx->pNames, pFetch); @@ -1857,7 +1857,7 @@ int32_t ctgLaunchGetTbHashsTask(SCtgTask* pTask) { return TSDB_CODE_SUCCESS; } - pTask->msgCtxs = taosArrayInit_s(pCtx->fetchNum, sizeof(SCtgMsgCtx), pCtx->fetchNum); + pTask->msgCtxs = taosArrayInit_s(sizeof(SCtgMsgCtx), pCtx->fetchNum); for (int32_t i = 0; i < pCtx->fetchNum; ++i) { SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 886eb8f387..f88701afe2 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -104,13 +104,13 @@ static void setDescResultIntoDataBlock(bool sysInfoUser, SSDataBlock* pBlock, in continue; } STR_TO_VARSTR(buf, pMeta->schema[i].name); - colDataAppend(pCol1, pBlock->info.rows, buf, false); + colDataSetVal(pCol1, pBlock->info.rows, buf, false); STR_TO_VARSTR(buf, tDataTypes[pMeta->schema[i].type].name); - colDataAppend(pCol2, pBlock->info.rows, buf, false); + colDataSetVal(pCol2, pBlock->info.rows, buf, false); int32_t bytes = getSchemaBytes(pMeta->schema + i); - colDataAppend(pCol3, pBlock->info.rows, (const char*)&bytes, false); + colDataSetVal(pCol3, pBlock->info.rows, (const char*)&bytes, false); STR_TO_VARSTR(buf, i >= pMeta->tableInfo.numOfColumns ? "TAG" : ""); - colDataAppend(pCol4, pBlock->info.rows, buf, false); + colDataSetVal(pCol4, pBlock->info.rows, buf, false); ++(pBlock->info.rows); } } @@ -255,7 +255,7 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbFName, S SColumnInfoData* pCol1 = taosArrayGet(pBlock->pDataBlock, 0); char buf1[SHOW_CREATE_DB_RESULT_FIELD1_LEN] = {0}; STR_TO_VARSTR(buf1, dbFName); - colDataAppend(pCol1, 0, buf1, false); + colDataSetVal(pCol1, 0, buf1, false); SColumnInfoData* pCol2 = taosArrayGet(pBlock->pDataBlock, 1); char buf2[SHOW_CREATE_DB_RESULT_FIELD2_LEN] = {0}; @@ -295,7 +295,7 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbFName, S (varDataLen(buf2)) = len; - colDataAppend(pCol2, 0, buf2, false); + colDataSetVal(pCol2, 0, buf2, false); } #define CHECK_LEADER(n) (row[n] && (fields[n].type == TSDB_DATA_TYPE_VARCHAR && strncasecmp(row[n], "leader", varDataLen((char *)row[n] - VARSTR_HEADER_SIZE)) == 0)) @@ -607,7 +607,7 @@ static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, SDbCfgInfo* p SColumnInfoData* pCol1 = taosArrayGet(pBlock->pDataBlock, 0); char buf1[SHOW_CREATE_TB_RESULT_FIELD1_LEN] = {0}; STR_TO_VARSTR(buf1, tbName); - colDataAppend(pCol1, 0, buf1, false); + colDataSetVal(pCol1, 0, buf1, false); SColumnInfoData* pCol2 = taosArrayGet(pBlock->pDataBlock, 1); char* buf2 = taosMemoryMalloc(SHOW_CREATE_TB_RESULT_FIELD2_LEN); @@ -645,7 +645,7 @@ static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, SDbCfgInfo* p varDataLen(buf2) = (len > 65535) ? 65535 : len; - colDataAppend(pCol2, 0, buf2, false); + colDataSetVal(pCol2, 0, buf2, false); taosMemoryFree(buf2); @@ -783,14 +783,14 @@ int32_t setLocalVariablesResultIntoDataBlock(SSDataBlock* pBlock) { char name[TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(name, pItem->name, TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE); SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, c++); - colDataAppend(pColInfo, i, name, false); + colDataSetVal(pColInfo, i, name, false); char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0}; int32_t valueLen = 0; cfgDumpItemValue(pItem, &value[VARSTR_HEADER_SIZE], TSDB_CONFIG_VALUE_LEN, &valueLen); varDataSetLen(value, valueLen); pColInfo = taosArrayGet(pBlock->pDataBlock, c++); - colDataAppend(pColInfo, i, value, false); + colDataSetVal(pColInfo, i, value, false); numOfRows++; } @@ -846,9 +846,9 @@ int32_t buildSelectResultDataBlock(SNodeList* pProjects, SSDataBlock* pBlock) { return TSDB_CODE_PAR_INVALID_SELECTED_EXPR; } else { if (((SValueNode*)pProj)->isNull) { - colDataAppend(taosArrayGet(pBlock->pDataBlock, index++), 0, NULL, true); + colDataSetVal(taosArrayGet(pBlock->pDataBlock, index++), 0, NULL, true); } else { - colDataAppend(taosArrayGet(pBlock->pDataBlock, index++), 0, nodesGetValueFromNode((SValueNode*)pProj), false); + colDataSetVal(taosArrayGet(pBlock->pDataBlock, index++), 0, nodesGetValueFromNode((SValueNode*)pProj), false); } } } diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index fdbbcad968..4302302d7a 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -1556,7 +1556,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { for (int32_t i = 0; i < rowNum; ++i) { SQueryExplainRowInfo *row = taosArrayGet(pCtx->rows, i); - colDataAppend(pInfoData, i, row->buf, false); + colDataSetVal(pInfoData, i, row->buf, false); } pBlock->info.rows = rowNum; diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 60e4e85fb8..d42b348fd8 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -182,10 +182,10 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, slotId); if (colDataIsNull_s(pSrc, pInfo->indexOfBufferedRes)) { - colDataAppendNULL(pDst, 0); + colDataSetNULL(pDst, 0); } else { char* p = colDataGetData(pSrc, pInfo->indexOfBufferedRes); - colDataAppend(pDst, 0, p, false); + colDataSetVal(pDst, 0, p, false); } } diff --git a/source/libs/executor/src/dataDeleter.c b/source/libs/executor/src/dataDeleter.c index 184b9e0148..dc3bf83a91 100644 --- a/source/libs/executor/src/dataDeleter.c +++ b/source/libs/executor/src/dataDeleter.c @@ -62,9 +62,6 @@ static void toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* pInp pEntry->numOfCols = taosArrayGetSize(pInput->pData->pDataBlock); pEntry->dataLen = sizeof(SDeleterRes); - // ASSERT(1 == pEntry->numOfRows); - // ASSERT(3 == pEntry->numOfCols); - pBuf->useSize = sizeof(SDataCacheEntry); SColumnInfoData* pColRes = (SColumnInfoData*)taosArrayGet(pInput->pData->pDataBlock, 0); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 639adb7ec6..62d1a3a66f 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -176,7 +176,6 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, in } pGroupResInfo->index = 0; - assert(pGroupResInfo->index <= getNumOfTotalRes(pGroupResInfo)); } void initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList) { @@ -340,10 +339,7 @@ int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, return code; } - ASSERT(nodeType(pNew) == QUERY_NODE_VALUE); SValueNode* pValue = (SValueNode*)pNew; - - ASSERT(pValue->node.resType.type == TSDB_DATA_TYPE_BOOL); *pQualified = pValue->datum.b; nodesDestroyNode(pNew); @@ -845,7 +841,7 @@ static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTa metaGetTableNameByUid(metaHandle, p1->uid, str); } - colDataAppend(pColInfo, i, str, false); + colDataSetVal(pColInfo, i, str, false); #if TAG_FILTER_DEBUG qDebug("tagfilter uid:%ld, tbname:%s", *uid, str + 2); #endif @@ -853,25 +849,26 @@ static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTa STagVal tagVal = {0}; tagVal.cid = pColInfo->info.colId; if (p1->pTagVal == NULL) { - colDataAppendNULL(pColInfo, i); + colDataSetNULL(pColInfo, i); } const char* p = metaGetTableTagVal(p1->pTagVal, pColInfo->info.type, &tagVal); if (p == NULL || (pColInfo->info.type == TSDB_DATA_TYPE_JSON && ((STag*)p)->nTag == 0)) { - colDataAppendNULL(pColInfo, i); + colDataSetNULL(pColInfo, i); } else if (pColInfo->info.type == TSDB_DATA_TYPE_JSON) { - colDataAppend(pColInfo, i, p, false); + colDataSetVal(pColInfo, i, p, false); } else if (IS_VAR_DATA_TYPE(pColInfo->info.type)) { - char* tmp = alloca(tagVal.nData + VARSTR_HEADER_SIZE + 1); + char* tmp = taosMemoryMalloc(tagVal.nData + VARSTR_HEADER_SIZE + 1); varDataSetLen(tmp, tagVal.nData); memcpy(tmp + VARSTR_HEADER_SIZE, tagVal.pData, tagVal.nData); - colDataAppend(pColInfo, i, tmp, false); + colDataSetVal(pColInfo, i, tmp, false); #if TAG_FILTER_DEBUG qDebug("tagfilter varch:%s", tmp + 2); #endif + taosMemoryFree(tmp); } else { - colDataAppend(pColInfo, i, (const char*)&tagVal.i64, false); + colDataSetVal(pColInfo, i, (const char*)&tagVal.i64, false); #if TAG_FILTER_DEBUG if (pColInfo->info.type == TSDB_DATA_TYPE_INT) { qDebug("tagfilter int:%d", *(int*)(&tagVal.i64)); @@ -1054,7 +1051,6 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, } if (!pTagCond) { // no tag filter condition exists, let's fetch all tables of this super table - ASSERT(pTagIndexCond == NULL); vnodeGetCtbIdList(pVnode, pScanNode->suid, pUidList); } else { // failed to find the result in the cache, let try to calculate the results @@ -1148,7 +1144,6 @@ int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, if (TSDB_CODE_SUCCESS == code) { REPLACE_NODE(pNew); } else { - taosMemoryFree(keyBuf); nodesDestroyList(groupNew); metaReaderClear(&mr); return code; @@ -1166,7 +1161,6 @@ int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, if (pValue->node.resType.type == TSDB_DATA_TYPE_JSON) { if (tTagIsJson(data)) { terrno = TSDB_CODE_QRY_JSON_IN_GROUP_ERROR; - taosMemoryFree(keyBuf); nodesDestroyList(groupNew); metaReaderClear(&mr); return terrno; @@ -1368,7 +1362,6 @@ void createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId) { if (!pFuncNode->pParameterList && (memcmp(pExprNode->_function.functionName, name, len) == 0) && pExprNode->_function.functionName[len] == 0) { pFuncNode->pParameterList = nodesMakeList(); - ASSERT(LIST_LENGTH(pFuncNode->pParameterList) == 0); SValueNode* res = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); if (NULL == res) { // todo handle error } else { @@ -1708,7 +1701,7 @@ static void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindo int64_t key = w->skey; while (key < ts) { // moving towards end key = taosTimeAdd(key, pInterval->sliding, pInterval->slidingUnit, pInterval->precision); - if (key >= ts) { + if (key > ts) { break; } @@ -1820,7 +1813,6 @@ uint64_t getTableGroupId(const STableListInfo* pTableList, uint64_t tableUid) { // TODO handle the group offset info, fix it, the rule of group output will be broken by this function int32_t tableListAddTableInfo(STableListInfo* pTableList, uint64_t uid, uint64_t gid) { if (pTableList->map == NULL) { - ASSERT(taosArrayGetSize(pTableList->pTableList) == 0); pTableList->map = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); } @@ -1970,7 +1962,6 @@ static int32_t sortTableGroup(STableListInfo* pTableListInfo) { int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* group, bool groupSort) { int32_t code = TSDB_CODE_SUCCESS; - ASSERT(pTableListInfo->map != NULL); bool groupByTbname = groupbyTbname(group); size_t numOfTables = taosArrayGetSize(pTableListInfo->pTableList); @@ -2027,7 +2018,6 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags } int32_t numOfTables = taosArrayGetSize(pTableListInfo->pTableList); - ASSERT(pTableListInfo->numOfOuputGroups == 1); int64_t st1 = taosGetTimestampUs(); pTaskInfo->cost.extractListTime = (st1 - st) / 1000.0; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index c599b479e6..151860d593 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -35,7 +35,6 @@ static void initRefPool() { } static int32_t doSetSMABlock(SOperatorInfo* pOperator, void* input, size_t numOfBlocks, int32_t type, char* id) { - ASSERT(pOperator != NULL); if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { if (pOperator->numOfDownstream == 0) { qError("failed to find stream scan operator to set the input data block, %s" PRIx64, id); @@ -78,27 +77,23 @@ static int32_t doSetSMABlock(SOperatorInfo* pOperator, void* input, size_t numOf } static int32_t doSetStreamOpOpen(SOperatorInfo* pOperator, char* id) { - { - ASSERT(pOperator != NULL); - if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { - if (pOperator->numOfDownstream == 0) { - qError("failed to find stream scan operator to set the input data block, %s" PRIx64, id); - return TSDB_CODE_APP_ERROR; - } - - if (pOperator->numOfDownstream > 1) { // not handle this in join query - qError("join not supported for stream block scan, %s" PRIx64, id); - return TSDB_CODE_APP_ERROR; - } - pOperator->status = OP_NOT_OPENED; - return doSetStreamOpOpen(pOperator->pDownstream[0], id); + if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { + if (pOperator->numOfDownstream == 0) { + qError("failed to find stream scan operator to set the input data block, %s" PRIx64, id); + return TSDB_CODE_APP_ERROR; } + + if (pOperator->numOfDownstream > 1) { // not handle this in join query + qError("join not supported for stream block scan, %s" PRIx64, id); + return TSDB_CODE_APP_ERROR; + } + pOperator->status = OP_NOT_OPENED; + return doSetStreamOpOpen(pOperator->pDownstream[0], id); } return 0; } static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t numOfBlocks, int32_t type, char* id) { - ASSERT(pOperator != NULL); if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { if (pOperator->numOfDownstream == 0) { qError("failed to find stream scan operator to set the input data block, %s" PRIx64, id); @@ -290,11 +285,15 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers) { static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const SArray* tableIdList, const char* idstr) { SArray* qa = taosArrayInit(4, sizeof(tb_uid_t)); + int32_t numOfUids = taosArrayGetSize(tableIdList); + if (numOfUids == 0) { + return qa; + } // let's discard the tables those are not created according to the queried super table. SMetaReader mr = {0}; metaReaderInit(&mr, pScanInfo->readHandle.meta, 0); - for (int32_t i = 0; i < taosArrayGetSize(tableIdList); ++i) { + for (int32_t i = 0; i < numOfUids; ++i) { uint64_t* id = (uint64_t*)taosArrayGet(tableIdList, i); int32_t code = metaGetTableEntryByUid(&mr, *id); @@ -359,7 +358,6 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo return code; } - // todo refactor STableList bool assignUid = false; size_t bufLen = (pScanInfo->pGroupTags != NULL) ? getTableTagsBufLen(pScanInfo->pGroupTags) : 0; char* keyBuf = NULL; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index e6ccb99b10..091c125df9 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -177,7 +177,7 @@ SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int32_t* currentPageId, i // set the number of rows in current disk page SResultRow* pResultRow = (SResultRow*)((char*)pData + pData->num); - memset((char*) pResultRow, 0, interBufSize); + memset((char*)pResultRow, 0, interBufSize); pResultRow->pageId = pageId; pResultRow->offset = (int32_t)pData->num; @@ -240,7 +240,6 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR // allocate a new buffer page if (pResult == NULL) { - ASSERT(pSup->resultRowSize > 0); pResult = getNewResultRow(pResultBuf, &pSup->currentPageId, pSup->resultRowSize); if (pResult == NULL) { T_LONG_JMP(pTaskInfo->env, terrno); @@ -310,7 +309,6 @@ static int32_t addNewWindowResultBuf(SResultRow* pWindowRes, SDiskbasedBuf* pRes pWindowRes->offset = (int32_t)pData->num; pData->num += size; - assert(pWindowRes->pageId >= 0); } return 0; @@ -322,13 +320,13 @@ void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow pColData->info.bytes = sizeof(int64_t); colInfoDataEnsureCapacity(pColData, 5, false); - colDataAppendInt64(pColData, 0, &pQueryWindow->skey); - colDataAppendInt64(pColData, 1, &pQueryWindow->ekey); + colDataSetInt64(pColData, 0, &pQueryWindow->skey); + colDataSetInt64(pColData, 1, &pQueryWindow->ekey); int64_t interval = 0; - colDataAppendInt64(pColData, 2, &interval); // this value may be variable in case of 'n' and 'y'. - colDataAppendInt64(pColData, 3, &pQueryWindow->skey); - colDataAppendInt64(pColData, 4, &pQueryWindow->ekey); + colDataSetInt64(pColData, 2, &interval); // this value may be variable in case of 'n' and 'y'. + colDataSetInt64(pColData, 3, &pQueryWindow->skey); + colDataSetInt64(pColData, 4, &pQueryWindow->ekey); } typedef struct { @@ -440,18 +438,18 @@ static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunc if (type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_UBIGINT) { int64_t v = pFuncParam->param.i; for (int32_t i = 0; i < numOfRows; ++i) { - colDataAppendInt64(pColInfo, i, &v); + colDataSetInt64(pColInfo, i, &v); } } else if (type == TSDB_DATA_TYPE_DOUBLE) { double v = pFuncParam->param.d; for (int32_t i = 0; i < numOfRows; ++i) { - colDataAppendDouble(pColInfo, i, &v); + colDataSetDouble(pColInfo, i, &v); } } else if (type == TSDB_DATA_TYPE_VARCHAR) { char* tmp = taosMemoryMalloc(pFuncParam->param.nLen + VARSTR_HEADER_SIZE); STR_WITH_SIZE_TO_VARSTR(tmp, pFuncParam->param.pz, pFuncParam->param.nLen); for (int32_t i = 0; i < numOfRows; ++i) { - colDataAppend(pColInfo, i, tmp, false); + colDataSetVal(pColInfo, i, tmp, false); } taosMemoryFree(tmp); } @@ -489,7 +487,6 @@ static int32_t doSetInputDataBlock(SExprSupp* pExprSup, SSDataBlock* pBlock, int // todo: refactor this if (fmIsImplicitTsFunc(pCtx[i].functionId) && (j == pOneExpr->base.numOfParams - 1)) { pInput->pPTS = pInput->pData[j]; // in case of merge function, this is not always the ts column data. - // ASSERT(pInput->pPTS->info.type == TSDB_DATA_TYPE_TIMESTAMP); } ASSERT(pInput->pData[j] != NULL); } else if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) { @@ -923,7 +920,7 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoD colDataSetNull_var(pDst, numOfRows); } else { char* p1 = colDataGetVarData(pDst, j); - colDataAppend(pDst, numOfRows, p1, false); + colDataSetVal(pDst, numOfRows, p1, false); } numOfRows += 1; j += 1; @@ -1038,8 +1035,6 @@ void doSetTableGroupOutputBuf(SOperatorInfo* pOperator, int32_t numOfOutput, uin SResultRow* pResultRow = doSetResultOutBufByKey(pAggInfo->aggSup.pResultBuf, pResultRowInfo, (char*)&groupId, sizeof(groupId), true, groupId, pTaskInfo, false, &pAggInfo->aggSup); - assert(pResultRow != NULL); - /* * not assign result buffer yet, add new result buffer * all group belong to one result set, and each group result has different group id so set the id to be one @@ -1117,7 +1112,7 @@ void copyResultrowToDataBlock(SExprInfo* pExprInfo, int32_t numOfExprs, SResultR SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo); for (int32_t k = 0; k < pRow->numOfRows; ++k) { - colDataAppend(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); + colDataSetVal(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); } } } @@ -1206,7 +1201,7 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS if (pBlock->info.rows <= 0 || pRow->numOfRows > pBlock->info.capacity) { qError("error in copy data to ssdatablock, existed rows in block:%d, rows in pRow:%d, capacity:%d, %s", - pBlock->info.rows, pRow->numOfRows, pBlock->info.capacity, GET_TASKID(pTaskInfo)); + pBlock->info.rows, pRow->numOfRows, pBlock->info.capacity, GET_TASKID(pTaskInfo)); T_LONG_JMP(pTaskInfo->env, TSDB_CODE_APP_ERROR); } else { break; @@ -1292,7 +1287,6 @@ void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SG // STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; // SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo; // -// assert(pQueryAttr->limit.offset == 0); // STimeWindow tw = *win; // getNextTimeWindow(pQueryAttr, &tw); // @@ -1307,7 +1301,6 @@ void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SG // tw = *win; // int32_t startPos = // getNextQualifiedWindow(pQueryAttr, &tw, pBlockInfo, pColInfoData->pData, binarySearchForKey, -1); -// assert(startPos >= 0); // // // set the abort info // pQueryAttr->pos = startPos; @@ -1342,11 +1335,6 @@ void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SG // static bool skipTimeInterval(STaskRuntimeEnv *pRuntimeEnv, TSKEY* start) { // STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; -// if (QUERY_IS_ASC_QUERY(pQueryAttr)) { -// assert(*start <= pRuntimeEnv->current->lastKey); -// } else { -// assert(*start >= pRuntimeEnv->current->lastKey); -// } // // // if queried with value filter, do NOT forward query start position // if (pQueryAttr->limit.offset <= 0 || pQueryAttr->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL || @@ -1360,8 +1348,6 @@ void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SG // value is // * not valid. otherwise, we only forward pQueryAttr->limit.offset number of points // */ -// assert(pRuntimeEnv->resultRowInfo.prevSKey == TSKEY_INITIAL_VAL); -// // STimeWindow w = TSWINDOW_INITIALIZER; // bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); // @@ -1431,8 +1417,6 @@ void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SG // tw = win; // int32_t startPos = // getNextQualifiedWindow(pQueryAttr, &tw, &blockInfo, pColInfoData->pData, binarySearchForKey, -1); -// assert(startPos >= 0); -// // // set the abort info // pQueryAttr->pos = startPos; // pTableQueryInfo->lastKey = ((TSKEY *)pColInfoData->pData)[startPos]; @@ -1454,10 +1438,6 @@ void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SG // } int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t num) { - if (p->pDownstream == NULL) { - assert(p->numOfDownstream == 0); - } - p->pDownstream = taosMemoryCalloc(1, num * POINTER_BYTES); if (p->pDownstream == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -1750,8 +1730,8 @@ int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaul int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t keyBufSize, const char* pKey) { - int32_t code = 0; -// _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + int32_t code = 0; + // _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pAggSup->currentPageId = -1; pAggSup->resultRowSize = getResultRowSize(pCtx, numOfOutput); @@ -1813,7 +1793,10 @@ int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo } void initResultSizeInfo(SResultInfo* pResultInfo, int32_t numOfRows) { - ASSERT(numOfRows != 0); + if (numOfRows == 0) { + numOfRows = 4096; + } + pResultInfo->capacity = numOfRows; pResultInfo->threshold = numOfRows * 0.75; @@ -1953,10 +1936,7 @@ _error: return NULL; } -void cleanupBasicInfo(SOptrBasicInfo* pInfo) { - assert(pInfo != NULL); - pInfo->pRes = blockDataDestroy(pInfo->pRes); -} +void cleanupBasicInfo(SOptrBasicInfo* pInfo) { pInfo->pRes = blockDataDestroy(pInfo->pRes); } static void freeItem(void* pItem) { void** p = pItem; @@ -2035,7 +2015,12 @@ int32_t extractTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNode, tDecoderClear(&mr.coder); tb_uid_t suid = mr.me.ctbEntry.suid; - metaGetTableEntryByUidCache(&mr, suid); + code = metaGetTableEntryByUidCache(&mr, suid); + if (code != TSDB_CODE_SUCCESS) { + metaReaderClear(&mr); + return terrno; + } + pSchemaInfo->sw = tCloneSSchemaWrapper(&mr.me.stbEntry.schemaRow); pSchemaInfo->tversion = mr.me.stbEntry.schemaTag.version; } else { @@ -2260,6 +2245,9 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pOperator = createCacherowsScanOperator(pScanNode, pHandle, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_PROJECT == type) { pOperator = createProjectOperatorInfo(NULL, (SProjectPhysiNode*)pPhyNode, pTaskInfo); + } else { + terrno = TSDB_CODE_INVALID_PARA; + return NULL; } if (pOperator != NULL) { @@ -2350,6 +2338,9 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pOptr = createTimeSliceOperatorInfo(ops[0], pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT == type) { pOptr = createEventwindowOperatorInfo(ops[0], pPhyNode, pTaskInfo); + } else { + terrno = TSDB_CODE_INVALID_PARA; + return NULL; } taosMemoryFree(ops); @@ -2587,7 +2578,6 @@ int32_t setOutputBuf(SStreamState* pState, STimeWindow* win, SResultRow** pResul return TSDB_CODE_OUT_OF_MEMORY; } *pResult = (SResultRow*)value; - ASSERT(*pResult); // set time window for current result (*pResult)->win = (*win); setResultRowInitCtx(*pResult, pCtx, numOfOutput, rowEntryInfoOffset); @@ -2616,9 +2606,9 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pStat for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) { SWinKey* pKey = taosArrayGet(pGroupResInfo->pRows, i); - int32_t size = 0; - void* pVal = NULL; - int32_t code = streamStateGet(pState, pKey, &pVal, &size); + int32_t size = 0; + void* pVal = NULL; + int32_t code = streamStateGet(pState, pKey, &pVal, &size); ASSERT(code == 0); SResultRow* pRow = (SResultRow*)pVal; doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset); @@ -2672,7 +2662,7 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pStat SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo); for (int32_t k = 0; k < pRow->numOfRows; ++k) { - colDataAppend(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); + colDataSetVal(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); } } } @@ -2765,7 +2755,7 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pSta SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo); for (int32_t k = 0; k < pRow->numOfRows; ++k) { - colDataAppend(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); + colDataSetVal(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); } } } diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index 483d94e8b1..9b5664492d 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -193,8 +193,6 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { return pResBlock; } } else if (pInfo->existNewGroupBlock) { // try next group - assert(pBlock != NULL); - blockDataCleanup(pResBlock); doHandleRemainBlockForNewGroupImpl(pOperator, pInfo, pResultInfo, pTaskInfo); @@ -411,7 +409,7 @@ TSKEY getPrevWindowTs(TSKEY ts, SInterval* pInterval) { } void setRowCell(SColumnInfoData* pCol, int32_t rowId, const SResultCellData* pCell) { - colDataAppend(pCol, rowId, pCell->pData, pCell->isNull); + colDataSetVal(pCol, rowId, pCell->pData, pCell->isNull); } SResultCellData* getResultCell(SResultRowData* pRaw, int32_t index) { @@ -905,7 +903,7 @@ static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo* } } else { if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || pCell->isNull) { - colDataAppendNULL(pColData, index); + colDataSetNULL(pColData, index); continue; } double* pDelta = taosArrayGet(pFillInfo->pLinearInfo->pDeltaVal, slotId); @@ -914,7 +912,7 @@ static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo* vCell += (*pDelta) * pFillInfo->pLinearInfo->winIndex; int64_t result = 0; SET_TYPED_DATA(&result, pCell->type, vCell); - colDataAppend(pColData, index, (const char*)&result, false); + colDataSetVal(pColData, index, (const char*)&result, false); } } pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, @@ -1033,23 +1031,23 @@ static void buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint64_ SColumnInfoData* pCalStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); SColumnInfoData* pCalEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); SColumnInfoData* pTbNameCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); - colDataAppend(pStartCol, pBlock->info.rows, (const char*)&start, false); - colDataAppend(pEndCol, pBlock->info.rows, (const char*)&end, false); - colDataAppendNULL(pUidCol, pBlock->info.rows); - colDataAppend(pGroupCol, pBlock->info.rows, (const char*)&groupId, false); - colDataAppendNULL(pCalStartCol, pBlock->info.rows); - colDataAppendNULL(pCalEndCol, pBlock->info.rows); + colDataSetVal(pStartCol, pBlock->info.rows, (const char*)&start, false); + colDataSetVal(pEndCol, pBlock->info.rows, (const char*)&end, false); + colDataSetNULL(pUidCol, pBlock->info.rows); + colDataSetVal(pGroupCol, pBlock->info.rows, (const char*)&groupId, false); + colDataSetNULL(pCalStartCol, pBlock->info.rows); + colDataSetNULL(pCalEndCol, pBlock->info.rows); SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); void* tbname = NULL; streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, groupId, &tbname); if (tbname == NULL) { - colDataAppendNULL(pTableCol, pBlock->info.rows); + colDataSetNULL(pTableCol, pBlock->info.rows); } else { char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN]; STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); - colDataAppend(pTableCol, pBlock->info.rows, (const char*)parTbName, false); + colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false); tdbFree(tbname); } diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 9fd8f7d3a2..cce182478d 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -204,7 +204,6 @@ static void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSData } static int32_t buildGroupKeys(void* pKey, const SArray* pGroupColVals) { - ASSERT(pKey != NULL); size_t numOfGroupCols = taosArrayGetSize(pGroupColVals); char* isNull = (char*)pKey; @@ -572,7 +571,6 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { } (*columnLen) += contentLen; - ASSERT(*columnLen >= 0); } (*rows) += 1; @@ -683,7 +681,6 @@ static int compareDataGroupInfo(const void* group1, const void* group2) { const SDataGroupInfo* pGroupInfo2 = group2; if (pGroupInfo1->groupId == pGroupInfo2->groupId) { - ASSERT(0); return 0; } @@ -960,7 +957,7 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { SColumnInfoData* pDestCol = taosArrayGet(pDest->pDataBlock, j); bool isNull = colDataIsNull(pSrcCol, pSrc->info.rows, rowIndex, NULL); char* pSrcData = colDataGetData(pSrcCol, rowIndex); - colDataAppend(pDestCol, pDest->info.rows, pSrcData, isNull); + colDataSetVal(pDestCol, pDest->info.rows, pSrcData, isNull); } pDest->info.rows++; } @@ -1005,7 +1002,7 @@ void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* pDestBlock->info.rows--; } else { void* pTbNameCol = taosArrayGet(pDestBlock->pDataBlock, UD_TABLE_NAME_COLUMN_INDEX); - colDataAppendNULL(pTbNameCol, pDestBlock->info.rows); + colDataSetNULL(pTbNameCol, pDestBlock->info.rows); pSrcBlock->info.parTbName[0] = 0; } diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 88ed9eccb3..31ff11eec5 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -24,6 +24,17 @@ #include "tmsg.h" #include "ttypes.h" +typedef struct SJoinRowCtx { + bool rowRemains; + int64_t ts; + SArray* leftRowLocations; + SArray* rightRowLocations; + SArray* leftCreatedBlocks; + SArray* rightCreatedBlocks; + int32_t leftRowIdx; + int32_t rightRowIdx; +} SJoinRowCtx; + typedef struct SJoinOperatorInfo { SSDataBlock* pRes; int32_t joinType; @@ -37,6 +48,8 @@ typedef struct SJoinOperatorInfo { int32_t rightPos; SColumnInfo rightCol; SNode* pCondAfterMerge; + + SJoinRowCtx rowCtx; } SJoinOperatorInfo; static void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode); @@ -196,10 +209,10 @@ static void mergeJoinJoinLeftRight(struct SOperatorInfo* pOperator, SSDataBlock* } if (colDataIsNull_s(pSrc, rowIndex)) { - colDataAppendNULL(pDst, currRow); + colDataSetNULL(pDst, currRow); } else { char* p = colDataGetData(pSrc, rowIndex); - colDataAppend(pDst, currRow, p, false); + colDataSetVal(pDst, currRow, p, false); } } } @@ -287,49 +300,107 @@ static int32_t mergeJoinGetDownStreamRowsEqualTimeStamp(SOperatorInfo* pOperator static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t timestamp, SSDataBlock* pRes, int32_t* nRows) { - SJoinOperatorInfo* pJoinInfo = pOperator->info; - SArray* leftRowLocations = taosArrayInit(8, sizeof(SRowLocation)); - SArray* leftCreatedBlocks = taosArrayInit(8, POINTER_BYTES); - - SArray* rightRowLocations = taosArrayInit(8, sizeof(SRowLocation)); - SArray* rightCreatedBlocks = taosArrayInit(8, POINTER_BYTES); int32_t code = TSDB_CODE_SUCCESS; - mergeJoinGetDownStreamRowsEqualTimeStamp(pOperator, 0, pJoinInfo->leftCol.slotId, pJoinInfo->pLeft, - pJoinInfo->leftPos, timestamp, leftRowLocations, leftCreatedBlocks); - mergeJoinGetDownStreamRowsEqualTimeStamp(pOperator, 1, pJoinInfo->rightCol.slotId, pJoinInfo->pRight, - pJoinInfo->rightPos, timestamp, rightRowLocations, rightCreatedBlocks); + SJoinOperatorInfo* pJoinInfo = pOperator->info; + SArray* leftRowLocations = NULL; + SArray* leftCreatedBlocks = NULL; + SArray* rightRowLocations = NULL; + SArray* rightCreatedBlocks = NULL; + int32_t leftRowIdx = 0; + int32_t rightRowIdx = 0; + int32_t i, j; + + if (pJoinInfo->rowCtx.rowRemains) { + leftRowLocations = pJoinInfo->rowCtx.leftRowLocations; + leftCreatedBlocks = pJoinInfo->rowCtx.leftCreatedBlocks; + rightRowLocations = pJoinInfo->rowCtx.rightRowLocations; + rightCreatedBlocks = pJoinInfo->rowCtx.rightCreatedBlocks; + leftRowIdx = pJoinInfo->rowCtx.leftRowIdx; + rightRowIdx = pJoinInfo->rowCtx.rightRowIdx; + } else { + leftRowLocations = taosArrayInit(8, sizeof(SRowLocation)); + leftCreatedBlocks = taosArrayInit(8, POINTER_BYTES); + rightRowLocations = taosArrayInit(8, sizeof(SRowLocation)); + rightCreatedBlocks = taosArrayInit(8, POINTER_BYTES); + + mergeJoinGetDownStreamRowsEqualTimeStamp(pOperator, 0, pJoinInfo->leftCol.slotId, pJoinInfo->pLeft, + pJoinInfo->leftPos, timestamp, leftRowLocations, leftCreatedBlocks); + mergeJoinGetDownStreamRowsEqualTimeStamp(pOperator, 1, pJoinInfo->rightCol.slotId, pJoinInfo->pRight, + pJoinInfo->rightPos, timestamp, rightRowLocations, rightCreatedBlocks); + } + size_t leftNumJoin = taosArrayGetSize(leftRowLocations); size_t rightNumJoin = taosArrayGetSize(rightRowLocations); - code = blockDataEnsureCapacity(pRes, *nRows + leftNumJoin * rightNumJoin); + uint32_t maxRowNum = *nRows + (leftNumJoin - leftRowIdx - 1) * rightNumJoin + rightNumJoin - rightRowIdx; + uint32_t limitRowNum = maxRowNum; + if (maxRowNum > pOperator->resultInfo.threshold) { + limitRowNum = pOperator->resultInfo.threshold; + if (!pJoinInfo->rowCtx.rowRemains) { + pJoinInfo->rowCtx.rowRemains = true; + pJoinInfo->rowCtx.ts = timestamp; + pJoinInfo->rowCtx.leftRowLocations = leftRowLocations; + pJoinInfo->rowCtx.rightRowLocations = rightRowLocations; + pJoinInfo->rowCtx.leftCreatedBlocks = leftCreatedBlocks; + pJoinInfo->rowCtx.rightCreatedBlocks = rightCreatedBlocks; + } + } + + code = blockDataEnsureCapacity(pRes, limitRowNum); if (code != TSDB_CODE_SUCCESS) { qError("%s can not ensure block capacity for join. left: %zu, right: %zu", GET_TASKID(pOperator->pTaskInfo), leftNumJoin, rightNumJoin); } + + if (code == TSDB_CODE_SUCCESS) { - for (int32_t i = 0; i < leftNumJoin; ++i) { - for (int32_t j = 0; j < rightNumJoin; ++j) { + bool done = false; + for (i = leftRowIdx; i < leftNumJoin; ++i, rightRowIdx = 0) { + for (j = rightRowIdx; j < rightNumJoin; ++j) { + if (*nRows >= limitRowNum) { + done = true; + break; + } + SRowLocation* leftRow = taosArrayGet(leftRowLocations, i); SRowLocation* rightRow = taosArrayGet(rightRowLocations, j); mergeJoinJoinLeftRight(pOperator, pRes, *nRows, leftRow->pDataBlock, leftRow->pos, rightRow->pDataBlock, rightRow->pos); ++*nRows; } + if (done) { + break; + } + } + + if (maxRowNum > pOperator->resultInfo.threshold) { + pJoinInfo->rowCtx.leftRowIdx = i; + pJoinInfo->rowCtx.rightRowIdx = j; } } - for (int i = 0; i < taosArrayGetSize(rightCreatedBlocks); ++i) { - SSDataBlock* pBlock = taosArrayGetP(rightCreatedBlocks, i); - blockDataDestroy(pBlock); + if (maxRowNum <= pOperator->resultInfo.threshold) { + for (int i = 0; i < taosArrayGetSize(rightCreatedBlocks); ++i) { + SSDataBlock* pBlock = taosArrayGetP(rightCreatedBlocks, i); + blockDataDestroy(pBlock); + } + taosArrayDestroy(rightCreatedBlocks); + taosArrayDestroy(rightRowLocations); + for (int i = 0; i < taosArrayGetSize(leftCreatedBlocks); ++i) { + SSDataBlock* pBlock = taosArrayGetP(leftCreatedBlocks, i); + blockDataDestroy(pBlock); + } + taosArrayDestroy(leftCreatedBlocks); + taosArrayDestroy(leftRowLocations); + + if (pJoinInfo->rowCtx.rowRemains) { + pJoinInfo->rowCtx.rowRemains = false; + pJoinInfo->rowCtx.leftRowLocations = NULL; + pJoinInfo->rowCtx.rightRowLocations = NULL; + pJoinInfo->rowCtx.leftCreatedBlocks = NULL; + pJoinInfo->rowCtx.rightCreatedBlocks = NULL; + } } - taosArrayDestroy(rightCreatedBlocks); - taosArrayDestroy(rightRowLocations); - for (int i = 0; i < taosArrayGetSize(leftCreatedBlocks); ++i) { - SSDataBlock* pBlock = taosArrayGetP(leftCreatedBlocks, i); - blockDataDestroy(pBlock); - } - taosArrayDestroy(leftCreatedBlocks); - taosArrayDestroy(leftRowLocations); return TSDB_CODE_SUCCESS; } @@ -379,9 +450,14 @@ static void doMergeJoinImpl(struct SOperatorInfo* pOperator, SSDataBlock* pRes) while (1) { int64_t leftTs = 0; int64_t rightTs = 0; - bool hasNextTs = mergeJoinGetNextTimestamp(pOperator, &leftTs, &rightTs); - if (!hasNextTs) { - break; + if (pJoinInfo->rowCtx.rowRemains) { + leftTs = pJoinInfo->rowCtx.ts; + rightTs = pJoinInfo->rowCtx.ts; + } else { + bool hasNextTs = mergeJoinGetNextTimestamp(pOperator, &leftTs, &rightTs); + if (!hasNextTs) { + break; + } } if (leftTs == rightTs) { @@ -389,12 +465,12 @@ static void doMergeJoinImpl(struct SOperatorInfo* pOperator, SSDataBlock* pRes) } else if ((asc && leftTs < rightTs) || (!asc && leftTs > rightTs)) { pJoinInfo->leftPos += 1; - if (pJoinInfo->leftPos >= pJoinInfo->pLeft->info.rows) { + if (pJoinInfo->leftPos >= pJoinInfo->pLeft->info.rows && pRes->info.rows < pOperator->resultInfo.threshold) { continue; } } else if ((asc && leftTs > rightTs) || (!asc && leftTs < rightTs)) { pJoinInfo->rightPos += 1; - if (pJoinInfo->rightPos >= pJoinInfo->pRight->info.rows) { + if (pJoinInfo->rightPos >= pJoinInfo->pRight->info.rows && pRes->info.rows < pOperator->resultInfo.threshold) { continue; } } diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index 5a221a0fbd..34ce67beb0 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -624,9 +624,9 @@ SSDataBlock* doGenerateSourceData(SOperatorInfo* pOperator) { int32_t type = pExpr[k].base.pParam[0].param.nType; if (TSDB_DATA_TYPE_NULL == type) { - colDataAppendNNULL(pColInfoData, 0, 1); + colDataSetNNULL(pColInfoData, 0, 1); } else { - colDataAppend(pColInfoData, 0, taosVariantGet(&pExpr[k].base.pParam[0].param, type), false); + colDataSetVal(pColInfoData, 0, taosVariantGet(&pExpr[k].base.pParam[0].param, type), false); } } @@ -666,9 +666,9 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc int32_t type = pExpr[k].base.pParam[0].param.nType; if (TSDB_DATA_TYPE_NULL == type) { - colDataAppendNNULL(pColInfoData, 0, 1); + colDataSetNNULL(pColInfoData, 0, 1); } else { - colDataAppend(pColInfoData, 0, taosVariantGet(&pExpr[k].base.pParam[0].param, type), false); + colDataSetVal(pColInfoData, 0, taosVariantGet(&pExpr[k].base.pParam[0].param, type), false); } } @@ -729,11 +729,11 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc int32_t type = pExpr[k].base.pParam[0].param.nType; if (TSDB_DATA_TYPE_NULL == type) { - colDataAppendNNULL(pColInfoData, offset, pSrcBlock->info.rows); + colDataSetNNULL(pColInfoData, offset, pSrcBlock->info.rows); } else { char* p = taosVariantGet(&pExpr[k].base.pParam[0].param, type); for (int32_t i = 0; i < pSrcBlock->info.rows; ++i) { - colDataAppend(pColInfoData, i + offset, p, false); + colDataSetVal(pColInfoData, i + offset, p, false); } } @@ -801,10 +801,10 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc for (int32_t f = 0; f < pSrcBlock->info.rows; ++f) { bool isNull = colDataIsNull_s(pInput, f); if (isNull) { - colDataAppendNULL(pOutput, pResult->info.rows + f); + colDataSetNULL(pOutput, pResult->info.rows + f); } else { char* data = colDataGetData(pInput, f); - colDataAppend(pOutput, pResult->info.rows + f, data, isNull); + colDataSetVal(pOutput, pResult->info.rows + f, data, isNull); } } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 9a56512683..d4071c8f63 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -267,7 +267,7 @@ bool applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo qDebug("current block ignore due to offset, current:%" PRId64 ", %s", pLimitInfo->remainOffset, id); return false; } else { - blockDataTrimFirstNRows(pBlock, pLimitInfo->remainOffset); + blockDataTrimFirstRows(pBlock, pLimitInfo->remainOffset); pLimitInfo->remainOffset = 0; } } @@ -455,7 +455,7 @@ static void doSetNullValue(SSDataBlock* pBlock, const SExprInfo* pExpr, int32_t int32_t dstSlotId = pExpr[j].base.resSchema.slotId; SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotId); - colDataAppendNNULL(pColInfoData, 0, pBlock->info.rows); + colDataSetNNULL(pColInfoData, 0, pBlock->info.rows); } } @@ -579,15 +579,15 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int bool isNullVal = (data == NULL) || (pColInfoData->info.type == TSDB_DATA_TYPE_JSON && tTagIsJsonNull(data)); if (isNullVal) { - colDataAppendNNULL(pColInfoData, 0, pBlock->info.rows); + colDataSetNNULL(pColInfoData, 0, pBlock->info.rows); } else if (pColInfoData->info.type != TSDB_DATA_TYPE_JSON) { - colDataAppendNItems(pColInfoData, 0, data, pBlock->info.rows); + colDataSetNItems(pColInfoData, 0, data, pBlock->info.rows); if (IS_VAR_DATA_TYPE(((const STagVal*)p)->type)) { taosMemoryFree(data); } } else { // todo opt for json tag for (int32_t i = 0; i < pBlock->info.rows; ++i) { - colDataAppend(pColInfoData, i, data, false); + colDataSetVal(pColInfoData, i, data, false); } } } @@ -613,7 +613,7 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, len, 1); colInfoDataEnsureCapacity(&infoData, 1, false); - colDataAppend(&infoData, 0, buf, false); + colDataSetVal(&infoData, 0, buf, false); SScalarParam srcParam = {.numOfRows = pBlock->info.rows, .columnData = &infoData}; SScalarParam param = {.columnData = pColInfoData}; @@ -1161,7 +1161,7 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32 SColumnInfoData* pDestCol = taosArrayGet(pResult->pDataBlock, j); bool isNull = colDataIsNull(pSrcCol, tmpBlock->info.rows, i, NULL); char* pSrcData = colDataGetData(pSrcCol, i); - colDataAppend(pDestCol, pResult->info.rows, pSrcData, isNull); + colDataSetVal(pDestCol, pResult->info.rows, pSrcData, isNull); } pResult->info.rows++; } @@ -1237,13 +1237,13 @@ static int32_t generateSessionScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSr qError("generate session scan range failed. rang start:%" PRIx64 ", end:%" PRIx64, startData[i], endData[i]); continue; } - colDataAppend(pDestStartCol, i, (const char*)&startWin.win.skey, false); - colDataAppend(pDestEndCol, i, (const char*)&endWin.win.ekey, false); + colDataSetVal(pDestStartCol, i, (const char*)&startWin.win.skey, false); + colDataSetVal(pDestEndCol, i, (const char*)&endWin.win.ekey, false); - colDataAppendNULL(pDestUidCol, i); - colDataAppend(pDestGpCol, i, (const char*)&groupId, false); - colDataAppendNULL(pDestCalStartTsCol, i); - colDataAppendNULL(pDestCalEndTsCol, i); + colDataSetNULL(pDestUidCol, i); + colDataSetVal(pDestGpCol, i, (const char*)&groupId, false); + colDataSetNULL(pDestCalStartTsCol, i); + colDataSetNULL(pDestCalEndTsCol, i); pDestBlock->info.rows++; } return TSDB_CODE_SUCCESS; @@ -1312,15 +1312,15 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS groupId = getGroupIdByData(pInfo, srcUid, srcStartTsCol[i], version); } TSKEY calStartTs = srcStartTsCol[i]; - colDataAppend(pCalStartTsCol, pDestBlock->info.rows, (const char*)(&calStartTs), false); + colDataSetVal(pCalStartTsCol, pDestBlock->info.rows, (const char*)(&calStartTs), false); STimeWindow win = getSlidingWindow(srcStartTsCol, srcEndTsCol, srcGp, &pInfo->interval, &pSrcBlock->info, &i, pInfo->partitionSup.needCalc); TSKEY calEndTs = srcStartTsCol[i - 1]; - colDataAppend(pCalEndTsCol, pDestBlock->info.rows, (const char*)(&calEndTs), false); - colDataAppend(pDeUidCol, pDestBlock->info.rows, (const char*)(&srcUid), false); - colDataAppend(pStartTsCol, pDestBlock->info.rows, (const char*)(&win.skey), false); - colDataAppend(pEndTsCol, pDestBlock->info.rows, (const char*)(&win.ekey), false); - colDataAppend(pGpCol, pDestBlock->info.rows, (const char*)(&groupId), false); + colDataSetVal(pCalEndTsCol, pDestBlock->info.rows, (const char*)(&calEndTs), false); + colDataSetVal(pDeUidCol, pDestBlock->info.rows, (const char*)(&srcUid), false); + colDataSetVal(pStartTsCol, pDestBlock->info.rows, (const char*)(&win.skey), false); + colDataSetVal(pEndTsCol, pDestBlock->info.rows, (const char*)(&win.ekey), false); + colDataSetVal(pGpCol, pDestBlock->info.rows, (const char*)(&groupId), false); pDestBlock->info.rows++; } return TSDB_CODE_SUCCESS; @@ -1435,13 +1435,13 @@ void appendOneRowToStreamSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKE SColumnInfoData* pCalStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); SColumnInfoData* pCalEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); - colDataAppend(pStartTsCol, pBlock->info.rows, (const char*)pStartTs, false); - colDataAppend(pEndTsCol, pBlock->info.rows, (const char*)pEndTs, false); - colDataAppend(pUidCol, pBlock->info.rows, (const char*)pUid, false); - colDataAppend(pGpCol, pBlock->info.rows, (const char*)pGp, false); - colDataAppend(pCalStartCol, pBlock->info.rows, (const char*)pStartTs, false); - colDataAppend(pCalEndCol, pBlock->info.rows, (const char*)pEndTs, false); - colDataAppend(pTableCol, pBlock->info.rows, (const char*)pTbName, pTbName == NULL); + colDataSetVal(pStartTsCol, pBlock->info.rows, (const char*)pStartTs, false); + colDataSetVal(pEndTsCol, pBlock->info.rows, (const char*)pEndTs, false); + colDataSetVal(pUidCol, pBlock->info.rows, (const char*)pUid, false); + colDataSetVal(pGpCol, pBlock->info.rows, (const char*)pGp, false); + colDataSetVal(pCalStartCol, pBlock->info.rows, (const char*)pStartTs, false); + colDataSetVal(pCalEndCol, pBlock->info.rows, (const char*)pEndTs, false); + colDataSetVal(pTableCol, pBlock->info.rows, (const char*)pTbName, pTbName == NULL); pBlock->info.rows++; } @@ -1528,7 +1528,7 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock // the required column does not exists in submit block, let's set it to be all null value if (!colExists) { SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, pColMatchInfo->dstSlotId); - colDataAppendNNULL(pDst, 0, pBlockInfo->rows); + colDataSetNNULL(pDst, 0, pBlockInfo->rows); } } @@ -1696,13 +1696,13 @@ static int32_t filterDelBlockByUid(SSDataBlock* pDst, const SSDataBlock* pSrc, S int32_t j = 0; for (int32_t i = 0; i < rows; i++) { if (taosHashGet(pReader->tbIdHash, &uidCol[i], sizeof(uint64_t))) { - colDataAppend(pDstStartCol, j, (const char*)&startCol[i], false); - colDataAppend(pDstEndCol, j, (const char*)&endCol[i], false); - colDataAppend(pDstUidCol, j, (const char*)&uidCol[i], false); + colDataSetVal(pDstStartCol, j, (const char*)&startCol[i], false); + colDataSetVal(pDstEndCol, j, (const char*)&endCol[i], false); + colDataSetVal(pDstUidCol, j, (const char*)&uidCol[i], false); - colDataAppendNULL(taosArrayGet(pDst->pDataBlock, GROUPID_COLUMN_INDEX), j); - colDataAppendNULL(taosArrayGet(pDst->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX), j); - colDataAppendNULL(taosArrayGet(pDst->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX), j); + colDataSetNULL(taosArrayGet(pDst->pDataBlock, GROUPID_COLUMN_INDEX), j); + colDataSetNULL(taosArrayGet(pDst->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX), j); + colDataSetNULL(taosArrayGet(pDst->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX), j); j++; } } @@ -1726,7 +1726,7 @@ static void setBlockGroupIdByUid(SStreamScanInfo* pInfo, SSDataBlock* pBlock) { if (!pInfo->partitionSup.needCalc) { for (int32_t i = 0; i < rows; i++) { uint64_t groupId = getGroupIdByUid(pInfo, uidCol[i]); - colDataAppend(pGpCol, i, (const char*)&groupId, false); + colDataSetVal(pGpCol, i, (const char*)&groupId, false); } } } @@ -2460,7 +2460,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { // refactor later if (fmIsScanPseudoColumnFunc(pExprInfo[j].pExpr->_function.functionId)) { STR_TO_VARSTR(str, mr.me.name); - colDataAppend(pDst, count, str, false); + colDataSetVal(pDst, count, str, false); } else { // it is a tag value STagVal val = {0}; val.cid = pExprInfo[j].base.pParam[0].pCol->colId; @@ -2472,7 +2472,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { } else { data = (char*)p; } - colDataAppend(pDst, count, data, + colDataSetVal(pDst, count, data, (data == NULL) || (pDst->info.type == TSDB_DATA_TYPE_JSON && tTagIsJsonNull(data))); if (pDst->info.type != TSDB_DATA_TYPE_JSON && p != NULL && IS_VAR_DATA_TYPE(((const STagVal*)p)->type) && @@ -3050,8 +3050,8 @@ int32_t tblCountScanGetInputs(SNodeList* groupTags, SName* tableName, STableCoun } } } else { - strncpy(supp->dbNameFilter, tNameGetDbNameP(tableName), TSDB_DB_NAME_LEN); - strncpy(supp->stbNameFilter, tNameGetTableName(tableName), TSDB_TABLE_NAME_LEN); + tstrncpy(supp->dbNameFilter, tNameGetDbNameP(tableName), TSDB_DB_NAME_LEN); + tstrncpy(supp->stbNameFilter, tNameGetTableName(tableName), TSDB_TABLE_NAME_LEN); } return TSDB_CODE_SUCCESS; } @@ -3129,7 +3129,7 @@ void fillTableCountScanDataBlock(STableCountScanSupp* pSupp, char* dbName, char* tstrncpy(varDataVal(varDbName), dbName, TSDB_DB_NAME_LEN); varDataSetLen(varDbName, strlen(dbName)); - colDataAppend(colInfoData, 0, varDbName, false); + colDataSetVal(colInfoData, 0, varDbName, false); } if (pSupp->stbNameSlotId != -1) { @@ -3138,15 +3138,15 @@ void fillTableCountScanDataBlock(STableCountScanSupp* pSupp, char* dbName, char* char varStbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; strncpy(varDataVal(varStbName), stbName, TSDB_TABLE_NAME_LEN); varDataSetLen(varStbName, strlen(stbName)); - colDataAppend(colInfoData, 0, varStbName, false); + colDataSetVal(colInfoData, 0, varStbName, false); } else { - colDataAppendNULL(colInfoData, 0); + colDataSetNULL(colInfoData, 0); } } if (pSupp->tbCountSlotId != -1) { SColumnInfoData* colInfoData = taosArrayGet(pRes->pDataBlock, pSupp->tbCountSlotId); - colDataAppend(colInfoData, 0, (char*)&count, false); + colDataSetVal(colInfoData, 0, (char*)&count, false); } pRes->info.rows = 1; } diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 98ef6b8a36..dc3ab79afb 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -101,11 +101,11 @@ void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); bool isNull = tsortIsNullVal(pTupleHandle, i); if (isNull) { - colDataAppendNULL(pColInfo, pBlock->info.rows); + colDataSetNULL(pColInfo, pBlock->info.rows); } else { char* pData = tsortGetValue(pTupleHandle, i); if (pData != NULL) { - colDataAppend(pColInfo, pBlock->info.rows, pData, false); + colDataSetVal(pColInfo, pBlock->info.rows, pData, false); } } } @@ -768,8 +768,6 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size pInfo->binfo.pRes = createDataBlockFromDescNode(pDescNode); int32_t rowSize = pInfo->binfo.pRes->info.rowSize; - ASSERT(rowSize < 100 * 1024 * 1024); - int32_t numOfOutputCols = 0; code = extractColMatchInfo(pMergePhyNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo); diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 1b2f135064..2a7edea290 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -133,7 +133,6 @@ static int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_ static SSDataBlock* buildInfoSchemaTableMetaBlock(char* tableName); static void destroySysScanOperator(void* param); static int32_t loadSysTableCallback(void* param, SDataBuf* pMsg, int32_t code); -static SSDataBlock* doFilterResult(SSDataBlock* pDataBlock, SFilterInfo* pFilterInfo); static __optSysFilter optSysGetFilterFunc(int32_t ctype, bool* reverse, bool* equal); static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smrSuperTable, @@ -838,21 +837,21 @@ static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, // table name pColInfoData = taosArrayGet(dataBlock->pDataBlock, 0); - colDataAppend(pColInfoData, numOfRows, tableName, false); + colDataSetVal(pColInfoData, numOfRows, tableName, false); // database name pColInfoData = taosArrayGet(dataBlock->pDataBlock, 1); - colDataAppend(pColInfoData, numOfRows, dbname, false); + colDataSetVal(pColInfoData, numOfRows, dbname, false); // super table name pColInfoData = taosArrayGet(dataBlock->pDataBlock, 2); - colDataAppend(pColInfoData, numOfRows, stableName, false); + colDataSetVal(pColInfoData, numOfRows, stableName, false); // tag name char tagName[TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(tagName, (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].name); pColInfoData = taosArrayGet(dataBlock->pDataBlock, 3); - colDataAppend(pColInfoData, numOfRows, tagName, false); + colDataSetVal(pColInfoData, numOfRows, tagName, false); // tag type int8_t tagType = (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].type; @@ -868,7 +867,7 @@ static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, (int32_t)(((*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); } varDataSetLen(tagTypeStr, tagTypeLen); - colDataAppend(pColInfoData, numOfRows, (char*)tagTypeStr, false); + colDataSetVal(pColInfoData, numOfRows, (char*)tagTypeStr, false); STagVal tagVal = {0}; tagVal.cid = (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].colId; @@ -908,7 +907,7 @@ static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, } } pColInfoData = taosArrayGet(dataBlock->pDataBlock, 5); - colDataAppend(pColInfoData, numOfRows, tagVarChar, + colDataSetVal(pColInfoData, numOfRows, tagVarChar, (tagData == NULL) || (tagType == TSDB_DATA_TYPE_JSON && tTagIsJsonNull(tagData))); taosMemoryFree(tagVarChar); ++numOfRows; @@ -969,7 +968,7 @@ static int32_t sysTableUserColsFillOneTableCols(const SSysTableScanInfo* pInfo, for (int32_t j = 6; j <= 8; ++j) { pColInfoData = taosArrayGet(dataBlock->pDataBlock, j); - colDataAppendNULL(pColInfoData, numOfRows); + colDataSetNULL(pColInfoData, numOfRows); } ++numOfRows; } @@ -1016,30 +1015,30 @@ int32_t buildDbTableInfoBlock(bool sysInfo, const SSDataBlock* p, const SSysTabl SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 0); STR_TO_VARSTR(n, pm->name); - colDataAppend(pColInfoData, numOfRows, n, false); + colDataSetVal(pColInfoData, numOfRows, n, false); // database name STR_TO_VARSTR(n, dbName); pColInfoData = taosArrayGet(p->pDataBlock, 1); - colDataAppend(pColInfoData, numOfRows, n, false); + colDataSetVal(pColInfoData, numOfRows, n, false); // create time pColInfoData = taosArrayGet(p->pDataBlock, 2); - colDataAppendNULL(pColInfoData, numOfRows); + colDataSetNULL(pColInfoData, numOfRows); // number of columns pColInfoData = taosArrayGet(p->pDataBlock, 3); - colDataAppend(pColInfoData, numOfRows, (char*)&pm->colNum, false); + colDataSetVal(pColInfoData, numOfRows, (char*)&pm->colNum, false); for (int32_t j = 4; j <= 8; ++j) { pColInfoData = taosArrayGet(p->pDataBlock, j); - colDataAppendNULL(pColInfoData, numOfRows); + colDataSetNULL(pColInfoData, numOfRows); } STR_TO_VARSTR(n, "SYSTEM_TABLE"); pColInfoData = taosArrayGet(p->pDataBlock, 9); - colDataAppend(pColInfoData, numOfRows, n, false); + colDataSetVal(pColInfoData, numOfRows, n, false); numOfRows += 1; } @@ -1107,22 +1106,22 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { // table name SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 0); - colDataAppend(pColInfoData, numOfRows, n, false); + colDataSetVal(pColInfoData, numOfRows, n, false); // database name pColInfoData = taosArrayGet(p->pDataBlock, 1); - colDataAppend(pColInfoData, numOfRows, dbname, false); + colDataSetVal(pColInfoData, numOfRows, dbname, false); // vgId pColInfoData = taosArrayGet(p->pDataBlock, 6); - colDataAppend(pColInfoData, numOfRows, (char*)&vgId, false); + colDataSetVal(pColInfoData, numOfRows, (char*)&vgId, false); int32_t tableType = mr.me.type; if (tableType == TSDB_CHILD_TABLE) { // create time int64_t ts = mr.me.ctbEntry.ctime; pColInfoData = taosArrayGet(p->pDataBlock, 2); - colDataAppend(pColInfoData, numOfRows, (char*)&ts, false); + colDataSetVal(pColInfoData, numOfRows, (char*)&ts, false); SMetaReader mr1 = {0}; metaReaderInit(&mr1, pInfo->readHandle.meta, META_READER_NOLOCK); @@ -1137,12 +1136,12 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { T_LONG_JMP(pTaskInfo->env, terrno); } pColInfoData = taosArrayGet(p->pDataBlock, 3); - colDataAppend(pColInfoData, numOfRows, (char*)&mr1.me.stbEntry.schemaRow.nCols, false); + colDataSetVal(pColInfoData, numOfRows, (char*)&mr1.me.stbEntry.schemaRow.nCols, false); // super table name STR_TO_VARSTR(n, mr1.me.name); pColInfoData = taosArrayGet(p->pDataBlock, 4); - colDataAppend(pColInfoData, numOfRows, n, false); + colDataSetVal(pColInfoData, numOfRows, n, false); metaReaderClear(&mr1); // table comment @@ -1150,59 +1149,59 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { if (mr.me.ctbEntry.commentLen > 0) { char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(comment, mr.me.ctbEntry.comment); - colDataAppend(pColInfoData, numOfRows, comment, false); + colDataSetVal(pColInfoData, numOfRows, comment, false); } else if (mr.me.ctbEntry.commentLen == 0) { char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(comment, ""); - colDataAppend(pColInfoData, numOfRows, comment, false); + colDataSetVal(pColInfoData, numOfRows, comment, false); } else { - colDataAppendNULL(pColInfoData, numOfRows); + colDataSetNULL(pColInfoData, numOfRows); } // uid pColInfoData = taosArrayGet(p->pDataBlock, 5); - colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.uid, false); + colDataSetVal(pColInfoData, numOfRows, (char*)&mr.me.uid, false); // ttl pColInfoData = taosArrayGet(p->pDataBlock, 7); - colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.ctbEntry.ttlDays, false); + colDataSetVal(pColInfoData, numOfRows, (char*)&mr.me.ctbEntry.ttlDays, false); STR_TO_VARSTR(n, "CHILD_TABLE"); } else if (tableType == TSDB_NORMAL_TABLE) { // create time pColInfoData = taosArrayGet(p->pDataBlock, 2); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ctime, false); + colDataSetVal(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ctime, false); // number of columns pColInfoData = taosArrayGet(p->pDataBlock, 3); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.schemaRow.nCols, false); + colDataSetVal(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.schemaRow.nCols, false); // super table name pColInfoData = taosArrayGet(p->pDataBlock, 4); - colDataAppendNULL(pColInfoData, numOfRows); + colDataSetNULL(pColInfoData, numOfRows); // table comment pColInfoData = taosArrayGet(p->pDataBlock, 8); if (mr.me.ntbEntry.commentLen > 0) { char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(comment, mr.me.ntbEntry.comment); - colDataAppend(pColInfoData, numOfRows, comment, false); + colDataSetVal(pColInfoData, numOfRows, comment, false); } else if (mr.me.ntbEntry.commentLen == 0) { char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(comment, ""); - colDataAppend(pColInfoData, numOfRows, comment, false); + colDataSetVal(pColInfoData, numOfRows, comment, false); } else { - colDataAppendNULL(pColInfoData, numOfRows); + colDataSetNULL(pColInfoData, numOfRows); } // uid pColInfoData = taosArrayGet(p->pDataBlock, 5); - colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.uid, false); + colDataSetVal(pColInfoData, numOfRows, (char*)&mr.me.uid, false); // ttl pColInfoData = taosArrayGet(p->pDataBlock, 7); - colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.ntbEntry.ttlDays, false); + colDataSetVal(pColInfoData, numOfRows, (char*)&mr.me.ntbEntry.ttlDays, false); STR_TO_VARSTR(n, "NORMAL_TABLE"); // impl later @@ -1211,7 +1210,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { metaReaderClear(&mr); pColInfoData = taosArrayGet(p->pDataBlock, 9); - colDataAppend(pColInfoData, numOfRows, n, false); + colDataSetVal(pColInfoData, numOfRows, n, false); if (++numOfRows >= pOperator->resultInfo.capacity) { p->info.rows = numOfRows; @@ -1285,22 +1284,22 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { // table name SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 0); - colDataAppend(pColInfoData, numOfRows, n, false); + colDataSetVal(pColInfoData, numOfRows, n, false); // database name pColInfoData = taosArrayGet(p->pDataBlock, 1); - colDataAppend(pColInfoData, numOfRows, dbname, false); + colDataSetVal(pColInfoData, numOfRows, dbname, false); // vgId pColInfoData = taosArrayGet(p->pDataBlock, 6); - colDataAppend(pColInfoData, numOfRows, (char*)&vgId, false); + colDataSetVal(pColInfoData, numOfRows, (char*)&vgId, false); int32_t tableType = pInfo->pCur->mr.me.type; if (tableType == TSDB_CHILD_TABLE) { // create time int64_t ts = pInfo->pCur->mr.me.ctbEntry.ctime; pColInfoData = taosArrayGet(p->pDataBlock, 2); - colDataAppend(pColInfoData, numOfRows, (char*)&ts, false); + colDataSetVal(pColInfoData, numOfRows, (char*)&ts, false); SMetaReader mr = {0}; metaReaderInit(&mr, pInfo->readHandle.meta, META_READER_NOLOCK); @@ -1318,12 +1317,12 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { // number of columns pColInfoData = taosArrayGet(p->pDataBlock, 3); - colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.stbEntry.schemaRow.nCols, false); + colDataSetVal(pColInfoData, numOfRows, (char*)&mr.me.stbEntry.schemaRow.nCols, false); // super table name STR_TO_VARSTR(n, mr.me.name); pColInfoData = taosArrayGet(p->pDataBlock, 4); - colDataAppend(pColInfoData, numOfRows, n, false); + colDataSetVal(pColInfoData, numOfRows, n, false); metaReaderClear(&mr); // table comment @@ -1331,64 +1330,64 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { if (pInfo->pCur->mr.me.ctbEntry.commentLen > 0) { char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(comment, pInfo->pCur->mr.me.ctbEntry.comment); - colDataAppend(pColInfoData, numOfRows, comment, false); + colDataSetVal(pColInfoData, numOfRows, comment, false); } else if (pInfo->pCur->mr.me.ctbEntry.commentLen == 0) { char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(comment, ""); - colDataAppend(pColInfoData, numOfRows, comment, false); + colDataSetVal(pColInfoData, numOfRows, comment, false); } else { - colDataAppendNULL(pColInfoData, numOfRows); + colDataSetNULL(pColInfoData, numOfRows); } // uid pColInfoData = taosArrayGet(p->pDataBlock, 5); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.uid, false); + colDataSetVal(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.uid, false); // ttl pColInfoData = taosArrayGet(p->pDataBlock, 7); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ctbEntry.ttlDays, false); + colDataSetVal(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ctbEntry.ttlDays, false); STR_TO_VARSTR(n, "CHILD_TABLE"); } else if (tableType == TSDB_NORMAL_TABLE) { // create time pColInfoData = taosArrayGet(p->pDataBlock, 2); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ctime, false); + colDataSetVal(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ctime, false); // number of columns pColInfoData = taosArrayGet(p->pDataBlock, 3); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.schemaRow.nCols, false); + colDataSetVal(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.schemaRow.nCols, false); // super table name pColInfoData = taosArrayGet(p->pDataBlock, 4); - colDataAppendNULL(pColInfoData, numOfRows); + colDataSetNULL(pColInfoData, numOfRows); // table comment pColInfoData = taosArrayGet(p->pDataBlock, 8); if (pInfo->pCur->mr.me.ntbEntry.commentLen > 0) { char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(comment, pInfo->pCur->mr.me.ntbEntry.comment); - colDataAppend(pColInfoData, numOfRows, comment, false); + colDataSetVal(pColInfoData, numOfRows, comment, false); } else if (pInfo->pCur->mr.me.ntbEntry.commentLen == 0) { char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(comment, ""); - colDataAppend(pColInfoData, numOfRows, comment, false); + colDataSetVal(pColInfoData, numOfRows, comment, false); } else { - colDataAppendNULL(pColInfoData, numOfRows); + colDataSetNULL(pColInfoData, numOfRows); } // uid pColInfoData = taosArrayGet(p->pDataBlock, 5); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.uid, false); + colDataSetVal(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.uid, false); // ttl pColInfoData = taosArrayGet(p->pDataBlock, 7); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ttlDays, false); + colDataSetVal(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ttlDays, false); STR_TO_VARSTR(n, "NORMAL_TABLE"); } pColInfoData = taosArrayGet(p->pDataBlock, 9); - colDataAppend(pColInfoData, numOfRows, n, false); + colDataSetVal(pColInfoData, numOfRows, n, false); if (++numOfRows >= pOperator->resultInfo.capacity) { p->info.rows = numOfRows; @@ -1611,7 +1610,7 @@ static void sysTableScanFillTbName(SOperatorInfo* pOperator, const SSysTableScan memcpy(varDataVal(varTbName), name, strlen(name)); varDataSetLen(varTbName, strlen(name)); - colDataAppendNItems(pColumnInfoData, 0, varTbName, pBlock->info.rows); + colDataSetNItems(pColumnInfoData, 0, varTbName, pBlock->info.rows); } doFilter(pBlock, pOperator->exprSupp.pFilterInfo, NULL); @@ -2174,14 +2173,14 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { tSerializeBlockDistInfo(varDataVal(p), len, &blockDistInfo); varDataSetLen(p, len); - colDataAppend(pColInfo, 0, p, false); + colDataSetVal(pColInfo, 0, p, false); taosMemoryFree(p); // make the valgrind happy that all memory buffer has been initialized already. if (slotId != 0) { SColumnInfoData* p1 = taosArrayGet(pBlock->pDataBlock, 0); int64_t v = 0; - colDataAppendInt64(p1, 0, &v); + colDataSetInt64(p1, 0, &v); } pBlock->info.rows = 1; @@ -2254,7 +2253,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi } pInfo->readHandle = *readHandle; - pInfo->uid = (pBlockScanNode->suid != 0)? pBlockScanNode->suid:pBlockScanNode->uid; + pInfo->uid = (pBlockScanNode->suid != 0) ? pBlockScanNode->suid : pBlockScanNode->uid; int32_t numOfCols = 0; SExprInfo* pExprInfo = createExprInfo(pBlockScanNode->pScanPseudoCols, NULL, &numOfCols); diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 778281f9b4..f4c8521b94 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -60,7 +60,7 @@ static void setNullRow(SSDataBlock* pBlock, SFillInfo* pFillInfo, int32_t rowInd setNotFillColumn(pFillInfo, pDstColInfo, rowIndex, i); } } else { - colDataAppendNULL(pDstColInfo, rowIndex); + colDataSetNULL(pDstColInfo, rowIndex); } } } @@ -69,19 +69,19 @@ static void doSetUserSpecifiedValue(SColumnInfoData* pDst, SVariant* pVar, int32 if (pDst->info.type == TSDB_DATA_TYPE_FLOAT) { float v = 0; GET_TYPED_DATA(v, float, pVar->nType, &pVar->i); - colDataAppend(pDst, rowIndex, (char*)&v, false); + colDataSetVal(pDst, rowIndex, (char*)&v, false); } else if (pDst->info.type == TSDB_DATA_TYPE_DOUBLE) { double v = 0; GET_TYPED_DATA(v, double, pVar->nType, &pVar->i); - colDataAppend(pDst, rowIndex, (char*)&v, false); + colDataSetVal(pDst, rowIndex, (char*)&v, false); } else if (IS_SIGNED_NUMERIC_TYPE(pDst->info.type)) { int64_t v = 0; GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i); - colDataAppend(pDst, rowIndex, (char*)&v, false); + colDataSetVal(pDst, rowIndex, (char*)&v, false); } else if (pDst->info.type == TSDB_DATA_TYPE_TIMESTAMP) { - colDataAppend(pDst, rowIndex, (const char*)¤tKey, false); + colDataSetVal(pDst, rowIndex, (const char*)¤tKey, false); } else { // varchar/nchar data - colDataAppendNULL(pDst, rowIndex); + colDataSetNULL(pDst, rowIndex); } } @@ -96,18 +96,18 @@ bool fillIfWindowPseudoColumn(SFillInfo* pFillInfo, SFillColInfo* pCol, SColumnI return false; } if (pCol->pExpr->base.pParam[0].pCol->colType == COLUMN_TYPE_WINDOW_START) { - colDataAppend(pDstColInfoData, rowIndex, (const char*)&pFillInfo->currentKey, false); + colDataSetVal(pDstColInfoData, rowIndex, (const char*)&pFillInfo->currentKey, false); return true; } else if (pCol->pExpr->base.pParam[0].pCol->colType == COLUMN_TYPE_WINDOW_END) { // TODO: include endpoint SInterval* pInterval = &pFillInfo->interval; int64_t windowEnd = taosTimeAdd(pFillInfo->currentKey, pInterval->interval, pInterval->intervalUnit, pInterval->precision); - colDataAppend(pDstColInfoData, rowIndex, (const char*)&windowEnd, false); + colDataSetVal(pDstColInfoData, rowIndex, (const char*)&windowEnd, false); return true; } else if (pCol->pExpr->base.pParam[0].pCol->colType == COLUMN_TYPE_WINDOW_DURATION) { // TODO: include endpoint - colDataAppend(pDstColInfoData, rowIndex, (const char*)&pFillInfo->interval.sliding, false); + colDataSetVal(pDstColInfoData, rowIndex, (const char*)&pFillInfo->interval.sliding, false); return true; } } @@ -163,7 +163,7 @@ static void doFillOneRow(SFillInfo* pFillInfo, SSDataBlock* pBlock, SSDataBlock* } else { SGroupKeys* pKey = taosArrayGet(pFillInfo->prev.pRowVal, i); if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || pKey->isNull) { - colDataAppendNULL(pDstCol, index); + colDataSetNULL(pDstCol, index); continue; } @@ -182,7 +182,7 @@ static void doFillOneRow(SFillInfo* pFillInfo, SSDataBlock* pBlock, SSDataBlock* point = (SPoint){.key = pFillInfo->currentKey, .val = &out}; taosGetLinearInterpolationVal(&point, type, &point1, &point2, type); - colDataAppend(pDstCol, index, (const char*)&out, false); + colDataSetVal(pDstCol, index, (const char*)&out, false); } } } @@ -217,9 +217,9 @@ static void doFillOneRow(SFillInfo* pFillInfo, SSDataBlock* pBlock, SSDataBlock* void doSetVal(SColumnInfoData* pDstCol, int32_t rowIndex, const SGroupKeys* pKey) { if (pKey->isNull) { - colDataAppendNULL(pDstCol, rowIndex); + colDataSetNULL(pDstCol, rowIndex); } else { - colDataAppend(pDstCol, rowIndex, pKey->pData, false); + colDataSetVal(pDstCol, rowIndex, pKey->pData, false); } } @@ -332,14 +332,14 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t char* src = colDataGetData(pSrc, pFillInfo->index); if (!colDataIsNull_s(pSrc, pFillInfo->index)) { - colDataAppend(pDst, index, src, false); + colDataSetVal(pDst, index, src, false); saveColData(pFillInfo->prev.pRowVal, i, src, false); if (pFillInfo->srcTsSlotId == dstSlotId) { pFillInfo->prev.key = *(int64_t*)src; } } else { // the value is null if (pDst->info.type == TSDB_DATA_TYPE_TIMESTAMP) { - colDataAppend(pDst, index, (const char*)&pFillInfo->currentKey, false); + colDataSetVal(pDst, index, (const char*)&pFillInfo->currentKey, false); } else { // i > 0 and data is null , do interpolation if (pFillInfo->type == TSDB_FILL_PREV) { SArray* p = FILL_IS_ASC_FILL(pFillInfo) ? pFillInfo->prev.pRowVal : pFillInfo->next.pRowVal; @@ -347,10 +347,10 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t doSetVal(pDst, index, pKey); } else if (pFillInfo->type == TSDB_FILL_LINEAR) { bool isNull = colDataIsNull_s(pSrc, pFillInfo->index); - colDataAppend(pDst, index, src, isNull); + colDataSetVal(pDst, index, src, isNull); saveColData(pFillInfo->prev.pRowVal, i, src, isNull); // todo: } else if (pFillInfo->type == TSDB_FILL_NULL || pFillInfo->type == TSDB_FILL_NULL_F) { - colDataAppendNULL(pDst, index); + colDataSetNULL(pDst, index); } else if (pFillInfo->type == TSDB_FILL_NEXT) { SArray* p = FILL_IS_ASC_FILL(pFillInfo) ? pFillInfo->next.pRowVal : pFillInfo->prev.pRowVal; SGroupKeys* pKey = taosArrayGet(p, i); diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c index bce8443353..911700be85 100644 --- a/source/libs/executor/src/timesliceoperator.c +++ b/source/libs/executor/src/timesliceoperator.c @@ -91,7 +91,6 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pSliceInfo->tsCol.slotId); SFillLinearInfo* pLinearInfo = taosArrayGet(pSliceInfo->pLinearInfo, i); - if (!IS_MATHABLE_TYPE(pColInfoData->info.type)) { continue; } @@ -100,7 +99,6 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo // TODO: optimize to ignore null values for linear interpolation. if (!pLinearInfo->isStartSet) { if (!colDataIsNull_s(pColInfoData, rowIndex)) { - pLinearInfo->start.key = *(int64_t*)colDataGetData(pTsCol, rowIndex); char* p = colDataGetData(pColInfoData, rowIndex); if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { @@ -144,10 +142,8 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo } } } - } - static FORCE_INLINE int32_t timeSliceEnsureBlockCapacity(STimeSliceOperatorInfo* pSliceInfo, SSDataBlock* pBlock) { if (pBlock->info.rows < pBlock->info.capacity) { return TSDB_CODE_SUCCESS; @@ -160,8 +156,8 @@ static FORCE_INLINE int32_t timeSliceEnsureBlockCapacity(STimeSliceOperatorInfo* return TSDB_CODE_SUCCESS; } - -static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock, bool beforeTs) { +static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock, + bool beforeTs) { int32_t rows = pResBlock->info.rows; timeSliceEnsureBlockCapacity(pSliceInfo, pResBlock); // todo set the correct primary timestamp column @@ -175,7 +171,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) { - colDataAppend(pDst, rows, (char*)&pSliceInfo->current, false); + colDataSetVal(pDst, rows, (char*)&pSliceInfo->current, false); continue; } else if (IS_BOOLEAN_TYPE(pExprInfo->base.resSchema.type)) { bool isFilled = true; @@ -187,7 +183,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp switch (pSliceInfo->fillType) { case TSDB_FILL_NULL: case TSDB_FILL_NULL_F: { - colDataAppendNULL(pDst, rows); + colDataSetNULL(pDst, rows); break; } @@ -198,15 +194,15 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp if (pDst->info.type == TSDB_DATA_TYPE_FLOAT) { float v = 0; GET_TYPED_DATA(v, float, pVar->nType, &pVar->i); - colDataAppend(pDst, rows, (char*)&v, false); + colDataSetVal(pDst, rows, (char*)&v, false); } else if (pDst->info.type == TSDB_DATA_TYPE_DOUBLE) { double v = 0; GET_TYPED_DATA(v, double, pVar->nType, &pVar->i); - colDataAppend(pDst, rows, (char*)&v, false); + colDataSetVal(pDst, rows, (char*)&v, false); } else if (IS_SIGNED_NUMERIC_TYPE(pDst->info.type)) { int64_t v = 0; GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i); - colDataAppend(pDst, rows, (char*)&v, false); + colDataSetVal(pDst, rows, (char*)&v, false); } break; } @@ -229,13 +225,13 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp } if (start.key == INT64_MIN || end.key == INT64_MIN) { - colDataAppendNULL(pDst, rows); + colDataSetNULL(pDst, rows); break; } current.val = taosMemoryCalloc(pLinearInfo->bytes, 1); taosGetLinearInterpolationVal(¤t, pLinearInfo->type, &start, &end, pLinearInfo->type); - colDataAppend(pDst, rows, (char*)current.val, false); + colDataSetVal(pDst, rows, (char*)current.val, false); taosMemoryFree(current.val); break; @@ -248,9 +244,9 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, srcSlot); if (pkey->isNull == false) { - colDataAppend(pDst, rows, pkey->pData, false); + colDataSetVal(pDst, rows, pkey->pData, false); } else { - colDataAppendNULL(pDst, rows); + colDataSetNULL(pDst, rows); } break; } @@ -263,9 +259,9 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, srcSlot); if (pkey->isNull == false) { - colDataAppend(pDst, rows, pkey->pData, false); + colDataSetVal(pDst, rows, pkey->pData, false); } else { - colDataAppendNULL(pDst, rows); + colDataSetNULL(pDst, rows); } break; } @@ -293,21 +289,21 @@ static void addCurrentRowToResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) { - colDataAppend(pDst, pResBlock->info.rows, (char*)&pSliceInfo->current, false); + colDataSetVal(pDst, pResBlock->info.rows, (char*)&pSliceInfo->current, false); } else if (IS_BOOLEAN_TYPE(pExprInfo->base.resSchema.type)) { bool isFilled = false; - colDataAppend(pDst, pResBlock->info.rows, (char*)&isFilled, false); + colDataSetVal(pDst, pResBlock->info.rows, (char*)&isFilled, false); } else { int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, srcSlot); if (colDataIsNull_s(pSrc, index)) { - colDataAppendNULL(pDst, pResBlock->info.rows); + colDataSetNULL(pDst, pResBlock->info.rows); continue; } char* v = colDataGetData(pSrc, index); - colDataAppend(pDst, pResBlock->info.rows, v, false); + colDataSetVal(pDst, pResBlock->info.rows, v, false); } } @@ -315,7 +311,6 @@ static void addCurrentRowToResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* return; } - static int32_t initPrevRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) { if (pInfo->pPrevRow != NULL) { return TSDB_CODE_SUCCESS; @@ -487,11 +482,12 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); if (nextTs > pSliceInfo->current) { while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { - if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, false) && pSliceInfo->fillType == TSDB_FILL_LINEAR) { + if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, false) && + pSliceInfo->fillType == TSDB_FILL_LINEAR) { break; } else { - pSliceInfo->current = - taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); + pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, + pInterval->precision); } } @@ -511,7 +507,8 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { doKeepLinearInfo(pSliceInfo, pBlock, i); while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) { - if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, true) && pSliceInfo->fillType == TSDB_FILL_LINEAR) { + if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, true) && + pSliceInfo->fillType == TSDB_FILL_LINEAR) { break; } else { pSliceInfo->current = @@ -606,14 +603,15 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode setOperatorInfo(pOperator, "TimeSliceOperator", QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC, false, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTimeslice, NULL, destroyTimeSliceOperatorInfo, optrDefaultBufFn, NULL); + pOperator->fpSet = + createOperatorFpSet(optrDummyOpenFn, doTimeslice, NULL, destroyTimeSliceOperatorInfo, optrDefaultBufFn, NULL); blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); code = appendDownstream(pOperator, &downstream, 1); return pOperator; - _error: +_error: taosMemoryFree(pInfo); taosMemoryFree(pOperator); pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index d379b9dfeb..09b7679f45 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2237,11 +2237,11 @@ static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pB SColumnInfoData* pCalEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); for (; (*pIndex) < size; (*pIndex)++) { SPullWindowInfo* pWin = taosArrayGet(array, (*pIndex)); - colDataAppend(pStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false); - colDataAppend(pEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false); - colDataAppend(pGroupId, pBlock->info.rows, (const char*)&pWin->groupId, false); - colDataAppend(pCalStartTs, pBlock->info.rows, (const char*)&pWin->calWin.skey, false); - colDataAppend(pCalEndTs, pBlock->info.rows, (const char*)&pWin->calWin.ekey, false); + colDataSetVal(pStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false); + colDataSetVal(pEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false); + colDataSetVal(pGroupId, pBlock->info.rows, (const char*)&pWin->groupId, false); + colDataSetVal(pCalStartTs, pBlock->info.rows, (const char*)&pWin->calWin.skey, false); + colDataSetVal(pCalEndTs, pBlock->info.rows, (const char*)&pWin->calWin.ekey, false); pBlock->info.rows++; } if ((*pIndex) == size) { @@ -3244,28 +3244,28 @@ void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlo } SSessionKey* res = tSimpleHashGetKey(*Ite, NULL); SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); - colDataAppend(pStartTsCol, pBlock->info.rows, (const char*)&res->win.skey, false); + colDataSetVal(pStartTsCol, pBlock->info.rows, (const char*)&res->win.skey, false); SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); - colDataAppend(pEndTsCol, pBlock->info.rows, (const char*)&res->win.skey, false); + colDataSetVal(pEndTsCol, pBlock->info.rows, (const char*)&res->win.skey, false); SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX); - colDataAppendNULL(pUidCol, pBlock->info.rows); + colDataSetNULL(pUidCol, pBlock->info.rows); SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); - colDataAppend(pGpCol, pBlock->info.rows, (const char*)&res->groupId, false); + colDataSetVal(pGpCol, pBlock->info.rows, (const char*)&res->groupId, false); SColumnInfoData* pCalStCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); - colDataAppendNULL(pCalStCol, pBlock->info.rows); + colDataSetNULL(pCalStCol, pBlock->info.rows); SColumnInfoData* pCalEdCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); - colDataAppendNULL(pCalEdCol, pBlock->info.rows); + colDataSetNULL(pCalEdCol, pBlock->info.rows); SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); void* tbname = NULL; streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, res->groupId, &tbname); if (tbname == NULL) { - colDataAppendNULL(pTableCol, pBlock->info.rows); + colDataSetNULL(pTableCol, pBlock->info.rows); } else { char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN]; STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); - colDataAppend(pTableCol, pBlock->info.rows, (const char*)parTbName, false); + colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false); tdbFree(tbname); } pBlock->info.rows += 1; diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 90f20f40b8..14de05cf6e 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -213,6 +213,7 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) { void* pPage = getNewBufPage(pHandle->pBuf, &pageId); if (pPage == NULL) { blockDataDestroy(p); + taosArrayDestroy(pPageIdList); return terrno; } @@ -320,10 +321,10 @@ static void appendOneRowToDataBlock(SSDataBlock* pBlock, const SSDataBlock* pSou bool isNull = colDataIsNull(pSrcColInfo, pSource->info.rows, *rowIndex, NULL); if (isNull) { - colDataAppend(pColInfo, pBlock->info.rows, NULL, true); + colDataSetVal(pColInfo, pBlock->info.rows, NULL, true); } else { char* pData = colDataGetData(pSrcColInfo, *rowIndex); - colDataAppend(pColInfo, pBlock->info.rows, pData, false); + colDataSetVal(pColInfo, pBlock->info.rows, pData, false); } } diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index 913aae6cc3..b9a696170a 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -110,13 +110,13 @@ SSDataBlock* getDummyBlock(SOperatorInfo* pOperator) { v = taosRand(); } - colDataAppend(pColInfo, i, reinterpret_cast(&v), false); + colDataSetVal(pColInfo, i, reinterpret_cast(&v), false); // sprintf(buf, "this is %d row", i); // STR_TO_VARSTR(b1, buf); // // SColumnInfoData* pColInfo2 = static_cast(TARRAY_GET_ELEM(pBlock->pDataBlock, 1)); - // colDataAppend(pColInfo2, i, b1, false); + // colDataSetVal(pColInfo2, i, b1, false); } pBlock->info.rows = pInfo->numOfRowsPerPage; @@ -155,7 +155,7 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator) { SColumnInfoData* pColInfo = static_cast(TARRAY_GET_ELEM(pBlock->pDataBlock, 0)); ts = (++pInfo->tsStart); - colDataAppend(pColInfo, i, reinterpret_cast(&ts), false); + colDataSetVal(pColInfo, i, reinterpret_cast(&ts), false); SColumnInfoData* pColInfo1 = static_cast(TARRAY_GET_ELEM(pBlock->pDataBlock, 1)); if (pInfo->type == data_desc) { @@ -166,13 +166,13 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator) { v = taosRand(); } - colDataAppend(pColInfo1, i, reinterpret_cast(&v), false); + colDataSetVal(pColInfo1, i, reinterpret_cast(&v), false); // sprintf(buf, "this is %d row", i); // STR_TO_VARSTR(b1, buf); // // SColumnInfoData* pColInfo2 = static_cast(TARRAY_GET_ELEM(pBlock->pDataBlock, 1)); - // colDataAppend(pColInfo2, i, b1, false); + // colDataSetVal(pColInfo2, i, b1, false); } pBlock->info.rows = pInfo->numOfRowsPerPage; diff --git a/source/libs/executor/test/sortTests.cpp b/source/libs/executor/test/sortTests.cpp index 62813453a1..f35d07804e 100644 --- a/source/libs/executor/test/sortTests.cpp +++ b/source/libs/executor/test/sortTests.cpp @@ -91,7 +91,7 @@ SSDataBlock* getSingleColDummyBlock(void* param) { return NULL; } varDataSetLen(str, len); - colDataAppend(pColInfo, i, reinterpret_cast(str), false); + colDataSetVal(pColInfo, i, reinterpret_cast(str), false); pBlock->info.hasVarCol = true; printf("nchar: %s\n", strOri); } else if (pInfo->type == TSDB_DATA_TYPE_BINARY) { @@ -99,12 +99,12 @@ SSDataBlock* getSingleColDummyBlock(void* param) { char str[64] = {0}; taosRandStr(varDataVal(str), size); varDataSetLen(str, size); - colDataAppend(pColInfo, i, reinterpret_cast(str), false); + colDataSetVal(pColInfo, i, reinterpret_cast(str), false); pBlock->info.hasVarCol = true; printf("binary: %s\n", varDataVal(str)); } else if (pInfo->type == TSDB_DATA_TYPE_DOUBLE || pInfo->type == TSDB_DATA_TYPE_FLOAT) { double v = rand_f2(); - colDataAppend(pColInfo, i, reinterpret_cast(&v), false); + colDataSetVal(pColInfo, i, reinterpret_cast(&v), false); printf("float: %f\n", v); } else { int64_t v = ++pInfo->startVal; @@ -115,7 +115,7 @@ SSDataBlock* getSingleColDummyBlock(void* param) { memcpy(result, (char*)(&v) + sizeof(int64_t) - tDataTypes[pInfo->type].bytes, tDataTypes[pInfo->type].bytes); } - colDataAppend(pColInfo, i, result, false); + colDataSetVal(pColInfo, i, result, false); printf("int: %" PRId64 "\n", v); taosMemoryFree(result); } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 8e52ae5f30..f838466afd 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -446,7 +446,7 @@ int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0; char* in = GET_ROWCELL_INTERBUF(pResInfo); - colDataAppend(pCol, pBlock->info.rows, in, pResInfo->isNullRes); + colDataSetVal(pCol, pBlock->info.rows, in, pResInfo->isNullRes); return pResInfo->numOfRes; } @@ -476,7 +476,7 @@ int32_t functionFinalizeWithResultBuf(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0; char* in = finalResult; - colDataAppend(pCol, pBlock->info.rows, in, pResInfo->isNullRes); + colDataSetVal(pCol, pBlock->info.rows, in, pResInfo->isNullRes); return pResInfo->numOfRes; } @@ -775,13 +775,13 @@ int32_t maxFunction(SqlFunctionCtx* pCtx) { static int32_t setNullSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t rowIndex); static int32_t setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuplePos* pTuplePos, - int32_t rowIndex); + int32_t rowIndex); int32_t minmaxFunctionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t code = TSDB_CODE_SUCCESS; SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx); - SMinmaxResInfo* pRes = GET_ROWCELL_INTERBUF(pEntryInfo); + SMinmaxResInfo* pRes = GET_ROWCELL_INTERBUF(pEntryInfo); int32_t slotId = pCtx->pExpr->base.resSchema.slotId; int32_t currentRow = pBlock->info.rows; @@ -795,32 +795,31 @@ int32_t minmaxFunctionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_BIGINT: ((int64_t*)pCol->pData)[currentRow] = pRes->v; -// colDataAppendInt64(pCol, currentRow, &pRes->v); break; case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_INT: - colDataAppendInt32(pCol, currentRow, (int32_t*)&pRes->v); + colDataSetInt32(pCol, currentRow, (int32_t*)&pRes->v); break; case TSDB_DATA_TYPE_USMALLINT: case TSDB_DATA_TYPE_SMALLINT: - colDataAppendInt16(pCol, currentRow, (int16_t*)&pRes->v); + colDataSetInt16(pCol, currentRow, (int16_t*)&pRes->v); break; case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_UTINYINT: case TSDB_DATA_TYPE_TINYINT: - colDataAppendInt8(pCol, currentRow, (int8_t*)&pRes->v); + colDataSetInt8(pCol, currentRow, (int8_t*)&pRes->v); break; case TSDB_DATA_TYPE_DOUBLE: - colDataAppendDouble(pCol, currentRow, (double*)&pRes->v); + colDataSetDouble(pCol, currentRow, (double*)&pRes->v); break; case TSDB_DATA_TYPE_FLOAT: { float v = GET_FLOAT_VAL(&pRes->v); - colDataAppendFloat(pCol, currentRow, &v); + colDataSetFloat(pCol, currentRow, &v); break; } } } else { - colDataAppendNULL(pCol, currentRow); + colDataSetNULL(pCol, currentRow); } if (pCtx->subsidiaries.num > 0) { @@ -844,7 +843,7 @@ int32_t setNullSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32 int32_t dstSlotId = pc->pExpr->base.resSchema.slotId; SColumnInfoData* pDstCol = taosArrayGet(pBlock->pDataBlock, dstSlotId); - colDataAppendNULL(pDstCol, rowIndex); + colDataSetNULL(pDstCol, rowIndex); } return TSDB_CODE_SUCCESS; @@ -874,9 +873,9 @@ int32_t setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STu SColumnInfoData* pDstCol = taosArrayGet(pBlock->pDataBlock, dstSlotId); if (nullList[j]) { - colDataAppendNULL(pDstCol, rowIndex); + colDataSetNULL(pDstCol, rowIndex); } else { - colDataAppend(pDstCol, rowIndex, pStart, false); + colDataSetVal(pDstCol, rowIndex, pStart, false); } pStart += pDstCol->info.bytes; } @@ -913,16 +912,14 @@ void appendSelectivityValue(SqlFunctionCtx* pCtx, int32_t rowIndex, int32_t pos) SColumnInfoData* pDstCol = taosArrayGet(pCtx->pDstBlock->pDataBlock, dstSlotId); if (colDataIsNull_s(pSrcCol, rowIndex) == true) { - colDataAppendNULL(pDstCol, pos); + colDataSetNULL(pDstCol, pos); } else { - colDataAppend(pDstCol, pos, pData, false); + colDataSetVal(pDstCol, pos, pData, false); } } } -void replaceTupleData(STuplePos* pDestPos, STuplePos* pSourcePos) { - *pDestPos = *pSourcePos; -} +void replaceTupleData(STuplePos* pDestPos, STuplePos* pSourcePos) { *pDestPos = *pSourcePos; } int32_t minMaxCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t isMinFunc) { SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); @@ -1301,7 +1298,7 @@ int32_t stddevPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); - colDataAppend(pCol, pBlock->info.rows, res, false); + colDataSetVal(pCol, pBlock->info.rows, res, false); taosMemoryFree(res); return pResInfo->numOfRes; @@ -1508,7 +1505,7 @@ int32_t leastSQRFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t currentRow = pBlock->info.rows; if (0 == pInfo->num) { - colDataAppendNULL(pCol, currentRow); + colDataSetNULL(pCol, currentRow); return 0; } @@ -1521,7 +1518,7 @@ int32_t leastSQRFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { double param02 = param[0][2] - param[1][2] * (param[0][1] / param[1][1]); if (0 == param00) { - colDataAppendNULL(pCol, currentRow); + colDataSetNULL(pCol, currentRow); return 0; } @@ -1537,7 +1534,7 @@ int32_t leastSQRFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { snprintf(varDataVal(buf), sizeof(buf) - VARSTR_HEADER_SIZE, "{slop:%.6lf, intercept:%.6lf}", param02, param12); varDataSetLen(buf, len); - colDataAppend(pCol, currentRow, buf, pResInfo->isNullRes); + colDataSetVal(pCol, currentRow, buf, pResInfo->isNullRes); return pResInfo->numOfRes; } @@ -1686,7 +1683,7 @@ int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SPercentileInfo* ppInfo = (SPercentileInfo*)GET_ROWCELL_INTERBUF(pResInfo); int32_t code = 0; - double v = 0; + double v = 0; tMemBucket* pMemBucket = ppInfo->pMemBucket; if (pMemBucket == NULL || pMemBucket->total == 0) { // check for null @@ -1717,7 +1714,7 @@ int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { } int32_t slotId = pCtx->pExpr->base.resSchema.slotId; - SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); varDataSetLen(buf, len); colDataAppend(pCol, pBlock->info.rows, buf, false); @@ -1742,7 +1739,6 @@ _fin_error: tMemBucketDestroy(pMemBucket); return code; - } bool getApercentileFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { @@ -2008,7 +2004,7 @@ int32_t apercentilePartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); - colDataAppend(pCol, pBlock->info.rows, res, false); + colDataSetVal(pCol, pBlock->info.rows, res, false); taosMemoryFree(res); return pResInfo->numOfRes; @@ -2103,7 +2099,7 @@ static void prepareBuf(SqlFunctionCtx* pCtx) { } static int32_t firstlastSaveTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SqlFunctionCtx* pCtx, - SFirstLastRes* pInfo) { + SFirstLastRes* pInfo) { int32_t code = TSDB_CODE_SUCCESS; if (pCtx->subsidiaries.num <= 0) { @@ -2156,7 +2152,8 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { } // All null data column, return directly. - if (pInput->colDataSMAIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows) && pInputCol->hasNull == true) { + if (pInput->colDataSMAIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows) && + pInputCol->hasNull == true) { // save selectivity value for column consisted of all null values int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); if (code != TSDB_CODE_SUCCESS) { @@ -2272,7 +2269,8 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { } // All null data column, return directly. - if (pInput->colDataSMAIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows) && pInputCol->hasNull == true) { + if (pInput->colDataSMAIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows) && + pInputCol->hasNull == true) { // save selectivity value for column consisted of all null values int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); if (code != TSDB_CODE_SUCCESS) { @@ -2366,7 +2364,7 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { } if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { - char* data = colDataGetData(pInputCol, chosen); + char* data = colDataGetData(pInputCol, chosen); int32_t code = doSaveCurrentVal(pCtx, i, cts, type, data); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2377,7 +2375,7 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { for (int32_t i = pInput->startRowIndex + round * 4; i < pInput->startRowIndex + pInput->numOfRows; ++i) { if (pResInfo->numOfRes == 0 || pInfo->ts < pts[i]) { - char* data = colDataGetData(pInputCol, i); + char* data = colDataGetData(pInputCol, i); int32_t code = doSaveCurrentVal(pCtx, i, pts[i], type, data); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2394,7 +2392,7 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { numOfElems++; if (pResInfo->numOfRes == 0 || pInfo->ts < pts[i]) { - char* data = colDataGetData(pInputCol, i); + char* data = colDataGetData(pInputCol, i); int32_t code = doSaveCurrentVal(pCtx, i, pts[i], type, data); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2441,7 +2439,7 @@ static int32_t firstLastTransferInfoImpl(SFirstLastRes* pInput, SFirstLastRes* p } static int32_t firstLastTransferInfo(SqlFunctionCtx* pCtx, SFirstLastRes* pInput, SFirstLastRes* pOutput, bool isFirst, - int32_t rowIndex) { + int32_t rowIndex) { if (TSDB_CODE_SUCCESS == firstLastTransferInfoImpl(pInput, pOutput, isFirst)) { int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pOutput); if (code != TSDB_CODE_SUCCESS) { @@ -2468,7 +2466,7 @@ static int32_t firstLastFunctionMergeImpl(SqlFunctionCtx* pCtx, bool isFirstQuer for (int32_t i = start; i < start + pInput->numOfRows; ++i) { char* data = colDataGetData(pCol, i); SFirstLastRes* pInputInfo = (SFirstLastRes*)varDataVal(data); - int32_t code = firstLastTransferInfo(pCtx, pInputInfo, pInfo, isFirstQuery, i); + int32_t code = firstLastTransferInfo(pCtx, pInputInfo, pInfo, isFirstQuery, i); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2494,7 +2492,7 @@ int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0; SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(pResInfo); - colDataAppend(pCol, pBlock->info.rows, pRes->buf, pRes->isNull || pResInfo->isNullRes); + colDataSetVal(pCol, pBlock->info.rows, pRes->buf, pRes->isNull || pResInfo->isNullRes); // handle selectivity code = setSelectivityValue(pCtx, pBlock, &pRes->pos, pBlock->info.rows); @@ -2519,7 +2517,7 @@ int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); - colDataAppend(pCol, pBlock->info.rows, res, false); + colDataSetVal(pCol, pBlock->info.rows, res, false); code = setSelectivityValue(pCtx, pBlock, &pRes->pos, pBlock->info.rows); taosMemoryFree(res); @@ -2700,7 +2698,7 @@ static int32_t doSetPrevVal(SDiffInfo* pDiffInfo, int32_t type, const char* pv, } static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, SColumnInfoData* pOutput, int32_t pos, - int32_t order, int64_t ts) { + int32_t order, int64_t ts) { int32_t factor = (order == TSDB_ORDER_ASC) ? 1 : -1; pDiffInfo->prevTs = ts; switch (type) { @@ -2710,7 +2708,7 @@ static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, if (delta < 0 && pDiffInfo->ignoreNegative) { colDataSetNull_f_s(pOutput, pos); } else { - colDataAppendInt64(pOutput, pos, &delta); + colDataSetInt64(pOutput, pos, &delta); } pDiffInfo->prev.i64 = v; @@ -2723,7 +2721,7 @@ static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, if (delta < 0 && pDiffInfo->ignoreNegative) { colDataSetNull_f_s(pOutput, pos); } else { - colDataAppendInt64(pOutput, pos, &delta); + colDataSetInt64(pOutput, pos, &delta); } pDiffInfo->prev.i64 = v; break; @@ -2734,7 +2732,7 @@ static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, if (delta < 0 && pDiffInfo->ignoreNegative) { colDataSetNull_f_s(pOutput, pos); } else { - colDataAppendInt64(pOutput, pos, &delta); + colDataSetInt64(pOutput, pos, &delta); } pDiffInfo->prev.i64 = v; break; @@ -2746,7 +2744,7 @@ static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, if (delta < 0 && pDiffInfo->ignoreNegative) { colDataSetNull_f_s(pOutput, pos); } else { - colDataAppendInt64(pOutput, pos, &delta); + colDataSetInt64(pOutput, pos, &delta); } pDiffInfo->prev.i64 = v; break; @@ -2757,7 +2755,7 @@ static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, if ((delta < 0 && pDiffInfo->ignoreNegative) || isinf(delta) || isnan(delta)) { // check for overflow colDataSetNull_f_s(pOutput, pos); } else { - colDataAppendDouble(pOutput, pos, &delta); + colDataSetDouble(pOutput, pos, &delta); } pDiffInfo->prev.d64 = v; break; @@ -2768,7 +2766,7 @@ static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, if ((delta < 0 && pDiffInfo->ignoreNegative) || isinf(delta) || isnan(delta)) { // check for overflow colDataSetNull_f_s(pOutput, pos); } else { - colDataAppendDouble(pOutput, pos, &delta); + colDataSetDouble(pOutput, pos, &delta); } pDiffInfo->prev.d64 = v; break; @@ -2908,8 +2906,8 @@ static STopBotRes* getTopBotOutputInfo(SqlFunctionCtx* pCtx) { return pRes; } -static int32_t doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSDataBlock* pSrcBlock, uint16_t type, - uint64_t uid, SResultRowEntryInfo* pEntryInfo, bool isTopQuery); +static int32_t doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSDataBlock* pSrcBlock, + uint16_t type, uint64_t uid, SResultRowEntryInfo* pEntryInfo, bool isTopQuery); static void addResult(SqlFunctionCtx* pCtx, STopBotResItem* pSourceItem, int16_t type, bool isTopQuery); @@ -2930,7 +2928,7 @@ int32_t topFunction(SqlFunctionCtx* pCtx) { } numOfElems++; - char* data = colDataGetData(pCol, i); + char* data = colDataGetData(pCol, i); int32_t code = doAddIntoResult(pCtx, data, i, pCtx->pSrcBlock, pRes->type, pInput->uid, pResInfo, true); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2964,7 +2962,7 @@ int32_t bottomFunction(SqlFunctionCtx* pCtx) { } numOfElems++; - char* data = colDataGetData(pCol, i); + char* data = colDataGetData(pCol, i); int32_t code = doAddIntoResult(pCtx, data, i, pCtx->pSrcBlock, pRes->type, pInput->uid, pResInfo, false); if (code != TSDB_CODE_SUCCESS) { return code; @@ -3016,7 +3014,7 @@ static int32_t topBotResComparFn(const void* p1, const void* p2, const void* par } int32_t doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSDataBlock* pSrcBlock, uint16_t type, - uint64_t uid, SResultRowEntryInfo* pEntryInfo, bool isTopQuery) { + uint64_t uid, SResultRowEntryInfo* pEntryInfo, bool isTopQuery) { STopBotRes* pRes = getTopBotOutputInfo(pCtx); SVariant val = {0}; @@ -3207,7 +3205,7 @@ static char* doLoadTupleData(SSerializeDataHandle* pHandle, const STuplePos* pPo if (pPage == NULL) { return NULL; } - char* p = pPage->data + pPos->offset; + char* p = pPage->data + pPos->offset; releaseBufPage(pHandle->pBuf, pPage); return p; } else { @@ -3236,13 +3234,13 @@ int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { // todo assign the tag value and the corresponding row data int32_t currentRow = pBlock->info.rows; if (pEntryInfo->numOfRes <= 0) { - colDataAppendNULL(pCol, currentRow); + colDataSetNULL(pCol, currentRow); code = setSelectivityValue(pCtx, pBlock, &pRes->nullTuplePos, currentRow); return code; } for (int32_t i = 0; i < pEntryInfo->numOfRes; ++i) { STopBotResItem* pItem = &pRes->pItems[i]; - colDataAppend(pCol, currentRow, (const char*)&pItem->v.i, false); + colDataSetVal(pCol, currentRow, (const char*)&pItem->v.i, false); #ifdef BUF_PAGE_DEBUG qDebug("page_finalize i:%d,item:%p,pageId:%d, offset:%d\n", i, pItem, pItem->tuplePos.pageId, pItem->tuplePos.offset); @@ -3459,7 +3457,7 @@ int32_t spreadPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); - colDataAppend(pCol, pBlock->info.rows, res, false); + colDataSetVal(pCol, pBlock->info.rows, res, false); taosMemoryFree(res); return pResInfo->numOfRes; @@ -3634,7 +3632,7 @@ int32_t elapsedPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); - colDataAppend(pCol, pBlock->info.rows, res, false); + colDataSetVal(pCol, pBlock->info.rows, res, false); taosMemoryFree(res); return pResInfo->numOfRes; @@ -3960,7 +3958,7 @@ int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { pInfo->bins[i].upper, pInfo->bins[i].percentage); } varDataSetLen(buf, len); - colDataAppend(pCol, currentRow, buf, false); + colDataSetVal(pCol, currentRow, buf, false); currentRow++; } @@ -3979,7 +3977,7 @@ int32_t histogramPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); - colDataAppend(pCol, pBlock->info.rows, res, false); + colDataSetVal(pCol, pBlock->info.rows, res, false); taosMemoryFree(res); return pResInfo->numOfRes; @@ -4198,7 +4196,7 @@ int32_t hllPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); - colDataAppend(pCol, pBlock->info.rows, res, false); + colDataSetVal(pCol, pBlock->info.rows, res, false); taosMemoryFree(res); return pResInfo->numOfRes; @@ -4331,7 +4329,7 @@ int32_t stateCountFunction(SqlFunctionCtx* pCtx) { numOfElems++; if (colDataIsNull_f(pInputCol->nullbitmap, i)) { - colDataAppendNULL(pOutput, i); + colDataSetNULL(pOutput, i); // handle selectivity if (pCtx->subsidiaries.num > 0) { appendSelectivityValue(pCtx, i, pCtx->offset + numOfElems - 1); @@ -4347,7 +4345,7 @@ int32_t stateCountFunction(SqlFunctionCtx* pCtx) { } else { pInfo->count = 0; } - colDataAppend(pOutput, pCtx->offset + numOfElems - 1, (char*)&output, false); + colDataSetVal(pOutput, pCtx->offset + numOfElems - 1, (char*)&output, false); // handle selectivity if (pCtx->subsidiaries.num > 0) { @@ -4393,7 +4391,7 @@ int32_t stateDurationFunction(SqlFunctionCtx* pCtx) { numOfElems++; if (colDataIsNull_f(pInputCol->nullbitmap, i)) { - colDataAppendNULL(pOutput, i); + colDataSetNULL(pOutput, i); // handle selectivity if (pCtx->subsidiaries.num > 0) { appendSelectivityValue(pCtx, i, pCtx->offset + numOfElems - 1); @@ -4413,7 +4411,7 @@ int32_t stateDurationFunction(SqlFunctionCtx* pCtx) { } else { pInfo->durationStart = 0; } - colDataAppend(pOutput, pCtx->offset + numOfElems - 1, (char*)&output, false); + colDataSetVal(pOutput, pCtx->offset + numOfElems - 1, (char*)&output, false); // handle selectivity if (pCtx->subsidiaries.num > 0) { @@ -4453,7 +4451,7 @@ int32_t csumFunction(SqlFunctionCtx* pCtx) { int32_t pos = startOffset + numOfElems; if (colDataIsNull_f(pInputCol->nullbitmap, i)) { - // colDataAppendNULL(pOutput, i); + // colDataSetNULL(pOutput, i); continue; } @@ -4462,21 +4460,21 @@ int32_t csumFunction(SqlFunctionCtx* pCtx) { int64_t v; GET_TYPED_DATA(v, int64_t, type, data); pSumRes->isum += v; - colDataAppend(pOutput, pos, (char*)&pSumRes->isum, false); + colDataSetVal(pOutput, pos, (char*)&pSumRes->isum, false); } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { uint64_t v; GET_TYPED_DATA(v, uint64_t, type, data); pSumRes->usum += v; - colDataAppend(pOutput, pos, (char*)&pSumRes->usum, false); + colDataSetVal(pOutput, pos, (char*)&pSumRes->usum, false); } else if (IS_FLOAT_TYPE(type)) { double v; GET_TYPED_DATA(v, double, type, data); pSumRes->dsum += v; // check for overflow if (isinf(pSumRes->dsum) || isnan(pSumRes->dsum)) { - colDataAppendNULL(pOutput, pos); + colDataSetNULL(pOutput, pos); } else { - colDataAppend(pOutput, pos, (char*)&pSumRes->dsum, false); + colDataSetVal(pOutput, pos, (char*)&pSumRes->dsum, false); } } @@ -4540,7 +4538,7 @@ int32_t mavgFunction(SqlFunctionCtx* pCtx) { int32_t pos = startOffset + numOfElems; if (colDataIsNull_f(pInputCol->nullbitmap, i)) { - // colDataAppendNULL(pOutput, i); + // colDataSetNULL(pOutput, i); continue; } @@ -4563,9 +4561,9 @@ int32_t mavgFunction(SqlFunctionCtx* pCtx) { double result = pInfo->sum / pInfo->numOfPoints; // check for overflow if (isinf(result) || isnan(result)) { - colDataAppendNULL(pOutput, pos); + colDataSetNULL(pOutput, pos); } else { - colDataAppend(pOutput, pos, (char*)&result, false); + colDataSetVal(pOutput, pos, (char*)&result, false); } // handle selectivity @@ -4668,7 +4666,7 @@ int32_t sampleFunction(SqlFunctionCtx* pCtx) { continue; } - char* data = colDataGetData(pInputCol, i); + char* data = colDataGetData(pInputCol, i); int32_t code = doReservoirSample(pCtx, pInfo, data, i); if (code != TSDB_CODE_SUCCESS) { return code; @@ -4688,7 +4686,7 @@ int32_t sampleFunction(SqlFunctionCtx* pCtx) { } int32_t sampleFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx); SSampleInfo* pInfo = getSampleOutputInfo(pCtx); @@ -4699,12 +4697,12 @@ int32_t sampleFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t currentRow = pBlock->info.rows; if (pInfo->numSampled == 0) { - colDataAppendNULL(pCol, currentRow); + colDataSetNULL(pCol, currentRow); code = setSelectivityValue(pCtx, pBlock, &pInfo->nullTuplePos, currentRow); return code; } for (int32_t i = 0; i < pInfo->numSampled; ++i) { - colDataAppend(pCol, currentRow + i, pInfo->data + i * pInfo->colBytes, false); + colDataSetVal(pCol, currentRow + i, pInfo->data + i * pInfo->colBytes, false); code = setSelectivityValue(pCtx, pBlock, &pInfo->tuplePos[i], currentRow + i); } @@ -4815,9 +4813,9 @@ int32_t tailFunction(SqlFunctionCtx* pCtx) { int32_t pos = startOffset + i; STailItem* pItem = pInfo->pItems[i]; if (pItem->isNull) { - colDataAppendNULL(pOutput, pos); + colDataSetNULL(pOutput, pos); } else { - colDataAppend(pOutput, pos, pItem->data, false); + colDataSetVal(pOutput, pos, pItem->data, false); } } @@ -4841,7 +4839,7 @@ int32_t tailFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t currentRow = pBlock->info.rows; for (int32_t i = 0; i < pEntryInfo->numOfRes; ++i) { STailItem* pItem = pInfo->pItems[i]; - colDataAppend(pCol, currentRow, pItem->data, false); + colDataSetVal(pCol, currentRow, pItem->data, false); currentRow += 1; } @@ -4935,12 +4933,12 @@ int32_t uniqueFunction(SqlFunctionCtx* pCtx) { for (int32_t i = 0; i < pInfo->numOfPoints; ++i) { SUniqueItem* pItem = (SUniqueItem*)(pInfo->pItems + i * (sizeof(SUniqueItem) + pInfo->colBytes)); if (pItem->isNull == true) { - colDataAppendNULL(pOutput, i); + colDataSetNULL(pOutput, i); } else { - colDataAppend(pOutput, i, pItem->data, false); + colDataSetVal(pOutput, i, pItem->data, false); } if (pTsOutput != NULL) { - colDataAppendInt64(pTsOutput, i, &pItem->timestamp); + colDataSetInt64(pTsOutput, i, &pItem->timestamp); } } @@ -5022,7 +5020,7 @@ int32_t modeFunction(SqlFunctionCtx* pCtx) { } numOfElems++; - char* data = colDataGetData(pInputCol, i); + char* data = colDataGetData(pInputCol, i); int32_t code = doModeAdd(pInfo, i, pCtx, data); if (code != TSDB_CODE_SUCCESS) { return code; @@ -5067,10 +5065,10 @@ int32_t modeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { if (maxCount != 0) { SModeItem* pResItem = (SModeItem*)(pInfo->pItems + resIndex * (sizeof(SModeItem) + pInfo->colBytes)); - colDataAppend(pCol, currentRow, pResItem->data, false); + colDataSetVal(pCol, currentRow, pResItem->data, false); code = setSelectivityValue(pCtx, pBlock, &pResItem->tuplePos, currentRow); } else { - colDataAppendNULL(pCol, currentRow); + colDataSetNULL(pCol, currentRow); code = setSelectivityValue(pCtx, pBlock, &pInfo->nullTuplePos, currentRow); } @@ -5444,6 +5442,7 @@ int32_t blockDistFunction(SqlFunctionCtx* pCtx) { if (pDistInfo->maxRows < p1.maxRows) { pDistInfo->maxRows = p1.maxRows; } + pDistInfo->numOfVgroups += (p1.numOfTables != 0 ? 1 : 0); for (int32_t i = 0; i < tListLen(pDistInfo->blockRowsHisto); ++i) { pDistInfo->blockRowsHisto[i] += p1.blockRowsHisto[i]; @@ -5473,6 +5472,7 @@ int32_t tSerializeBlockDistInfo(void* buf, int32_t bufLen, const STableBlockDist if (tEncodeI32(&encoder, pInfo->defMinRows) < 0) return -1; if (tEncodeU32(&encoder, pInfo->numOfInmemRows) < 0) return -1; if (tEncodeU32(&encoder, pInfo->numOfSmallBlocks) < 0) return -1; + if (tEncodeU32(&encoder, pInfo->numOfVgroups) < 0) return -1; for (int32_t i = 0; i < tListLen(pInfo->blockRowsHisto); ++i) { if (tEncodeI32(&encoder, pInfo->blockRowsHisto[i]) < 0) return -1; @@ -5505,6 +5505,7 @@ int32_t tDeserializeBlockDistInfo(void* buf, int32_t bufLen, STableBlockDistInfo if (tDecodeI32(&decoder, &pInfo->defMinRows) < 0) return -1; if (tDecodeU32(&decoder, &pInfo->numOfInmemRows) < 0) return -1; if (tDecodeU32(&decoder, &pInfo->numOfSmallBlocks) < 0) return -1; + if (tDecodeU32(&decoder, &pInfo->numOfVgroups) < 0) return -1; for (int32_t i = 0; i < tListLen(pInfo->blockRowsHisto); ++i) { if (tDecodeI32(&decoder, &pInfo->blockRowsHisto[i]) < 0) return -1; @@ -5541,7 +5542,7 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { pData->numOfBlocks, pData->totalSize / 1024.0, averageSize / 1024.0, compRatio, '%'); varDataSetLen(st, len); - colDataAppend(pColInfo, row++, st, false); + colDataSetVal(pColInfo, row++, st, false); int64_t avgRows = 0; if (pData->numOfBlocks > 0) { @@ -5553,18 +5554,18 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { pData->totalRows, pData->numOfInmemRows, pData->minRows, pData->maxRows, avgRows); varDataSetLen(st, len); - colDataAppend(pColInfo, row++, st, false); + colDataSetVal(pColInfo, row++, st, false); len = sprintf(st + VARSTR_HEADER_SIZE, "Total_Tables=[%d] Total_Files=[%d] Total_Vgroups=[%d]", pData->numOfTables, pData->numOfFiles, pData->numOfVgroups); varDataSetLen(st, len); - colDataAppend(pColInfo, row++, st, false); + colDataSetVal(pColInfo, row++, st, false); len = sprintf(st + VARSTR_HEADER_SIZE, "--------------------------------------------------------------------------------"); varDataSetLen(st, len); - colDataAppend(pColInfo, row++, st, false); + colDataSetVal(pColInfo, row++, st, false); int32_t maxVal = 0; int32_t minVal = INT32_MAX; @@ -5603,7 +5604,7 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { } varDataSetLen(st, len); - colDataAppend(pColInfo, row++, st, false); + colDataSetVal(pColInfo, row++, st, false); } return TSDB_CODE_SUCCESS; @@ -5664,13 +5665,13 @@ int32_t derivativeFunction(SqlFunctionCtx* pCtx) { if (pDerivInfo->ignoreNegative && r < 0) { } else { if (isinf(r) || isnan(r)) { - colDataAppendNULL(pOutput, pos); + colDataSetNULL(pOutput, pos); } else { - colDataAppend(pOutput, pos, (const char*)&r, false); + colDataSetVal(pOutput, pos, (const char*)&r, false); } if (pTsOutput != NULL) { - colDataAppendInt64(pTsOutput, pos, &tsList[i]); + colDataSetInt64(pTsOutput, pos, &tsList[i]); } // handle selectivity @@ -5705,13 +5706,13 @@ int32_t derivativeFunction(SqlFunctionCtx* pCtx) { if (pDerivInfo->ignoreNegative && r < 0) { } else { if (isinf(r) || isnan(r)) { - colDataAppendNULL(pOutput, pos); + colDataSetNULL(pOutput, pos); } else { - colDataAppend(pOutput, pos, (const char*)&r, false); + colDataSetVal(pOutput, pos, (const char*)&r, false); } if (pTsOutput != NULL) { - colDataAppendInt64(pTsOutput, pos, &pDerivInfo->prevTs); + colDataSetInt64(pTsOutput, pos, &pDerivInfo->prevTs); } // handle selectivity @@ -5842,7 +5843,7 @@ int32_t irateFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SRateInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); double result = doCalcRate(pInfo, (double)TSDB_TICK_PER_SECOND(pCtx->param[1].param.i)); - colDataAppend(pCol, pBlock->info.rows, (const char*)&result, pResInfo->isNullRes); + colDataSetVal(pCol, pBlock->info.rows, (const char*)&result, pResInfo->isNullRes); return pResInfo->numOfRes; } @@ -5893,7 +5894,7 @@ int32_t groupKeyFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { if (pInfo->hasResult) { int32_t currentRow = pBlock->info.rows; for (; currentRow < pBlock->info.rows + pResInfo->numOfRes; ++currentRow) { - colDataAppend(pCol, currentRow, pInfo->data, pInfo->isNull ? true : false); + colDataSetVal(pCol, currentRow, pInfo->data, pInfo->isNull ? true : false); } } else { pResInfo->numOfRes = 0; diff --git a/source/libs/function/src/detail/tavgfunction.c b/source/libs/function/src/detail/tavgfunction.c index 3a70a65ec4..50df1b5067 100644 --- a/source/libs/function/src/detail/tavgfunction.c +++ b/source/libs/function/src/detail/tavgfunction.c @@ -847,7 +847,7 @@ int32_t avgPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); - colDataAppend(pCol, pBlock->info.rows, res, false); + colDataSetVal(pCol, pBlock->info.rows, res, false); taosMemoryFree(res); return pResInfo->numOfRes; diff --git a/source/libs/function/src/tfunctionInt.c b/source/libs/function/src/tfunctionInt.c index edfd866010..e8041d1704 100644 --- a/source/libs/function/src/tfunctionInt.c +++ b/source/libs/function/src/tfunctionInt.c @@ -47,11 +47,11 @@ int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num, SSDataBlock* pResBlock SResultRowEntryInfo* pResInfo = GET_RES_INFO(&pCtx[i]); if (pResInfo->numOfRes == 0) { for (int32_t j = 0; j < pResInfo->numOfRes; ++j) { - colDataAppend(pCol, j, NULL, true); // TODO add set null data api + colDataSetVal(pCol, j, NULL, true); // TODO add set null data api } } else { for (int32_t j = 0; j < pResInfo->numOfRes; ++j) { - colDataAppend(pCol, j, GET_ROWCELL_INTERBUF(pResInfo), false); + colDataSetVal(pCol, j, GET_ROWCELL_INTERBUF(pResInfo), false); } } } diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index b9e72847a1..8f5cd070dc 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -864,13 +864,13 @@ int32_t convertScalarParamToDataBlock(SScalarParam *input, int32_t numOfCols, SS int expandRows = numOfRows - startRow; bool isNull = colDataIsNull_s(pColInfoData, (input+i)->numOfRows - 1); if (isNull) { - colDataAppendNNULL(pDest, startRow, expandRows); + colDataSetNNULL(pDest, startRow, expandRows); } else { char* src = colDataGetData(pColInfoData, (input + i)->numOfRows - 1); for (int j = 0; j < expandRows; ++j) { - colDataAppend(pDest, startRow+j, src, false); + colDataSetVal(pDest, startRow+j, src, false); } - //colDataAppendNItems(pColInfoData, startRow, data, expandRows); + //colDataSetNItems(pColInfoData, startRow, data, expandRows); } } } diff --git a/source/libs/function/test/runUdf.c b/source/libs/function/test/runUdf.c index b5d1b1576b..aa8b88b738 100644 --- a/source/libs/function/test/runUdf.c +++ b/source/libs/function/test/runUdf.c @@ -55,7 +55,7 @@ int scalarFuncTest() { SColumnInfoData *pCol = taosArrayGet(pBlock->pDataBlock, 0); for (int32_t j = 0; j < pBlock->info.rows; ++j) { - colDataAppendInt32(pCol, j, &j); + colDataSetInt32(pCol, j, &j); } SScalarParam input = {0}; @@ -99,7 +99,7 @@ int aggregateFuncTest() { SColumnInfoData *pColInfo = bdGetColumnInfoData(pBlock, 0); for (int32_t j = 0; j < pBlock->info.rows; ++j) { - colDataAppendInt32(pColInfo, j, &j); + colDataSetInt32(pColInfo, j, &j); } SUdfInterBuf buf = {0}; diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index bd06572cb4..f198b73c7d 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -76,6 +76,7 @@ typedef struct SQWDebug { bool lockEnable; bool statusEnable; bool dumpEnable; + bool forceStop; bool sleepSimulate; bool deadSimulate; bool redirectSimulate; @@ -248,6 +249,7 @@ typedef struct SQWorkerMgmt { #define QW_QUERY_RUNNING(ctx) (QW_GET_PHASE(ctx) == QW_PHASE_PRE_QUERY || QW_GET_PHASE(ctx) == QW_PHASE_PRE_CQUERY) #define QW_FETCH_RUNNING(ctx) ((ctx)->inFetch) +#define QW_QUERY_NOT_STARTED(ctx) (QW_GET_PHASE(ctx) == -1) #define QW_SET_QTID(id, qId, tId, eId) \ do { \ diff --git a/source/libs/qworker/src/qwDbg.c b/source/libs/qworker/src/qwDbg.c index 7a755cd36f..0ab501ddd5 100644 --- a/source/libs/qworker/src/qwDbg.c +++ b/source/libs/qworker/src/qwDbg.c @@ -9,11 +9,13 @@ #include "tmsg.h" #include "tname.h" -SQWDebug gQWDebug = {.statusEnable = true, +SQWDebug gQWDebug = {.lockEnable = false, + .statusEnable = true, .dumpEnable = false, .redirectSimulate = false, .deadSimulate = false, - .sleepSimulate = false}; + .sleepSimulate = false, + .forceStop = false}; int32_t qwDbgValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus, bool *ignore) { if (!gQWDebug.statusEnable) { @@ -306,6 +308,12 @@ int32_t qwDbgEnableDebug(char *option) { return TSDB_CODE_SUCCESS; } + if (0 == strcasecmp(option, "forceStop")) { + gQWDebug.forceStop = true; + qError("qw forceStop debug enabled"); + return TSDB_CODE_SUCCESS; + } + qError("invalid qw debug option:%s", option); return TSDB_CODE_APP_ERROR; diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index b120fecd9d..ec4049a3eb 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -18,6 +18,50 @@ SQWorkerMgmt gQwMgmt = { .qwNum = 0, }; +int32_t qwStopAllTasks(SQWorker *mgmt) { + uint64_t qId, tId, sId; + int32_t eId; + int64_t rId = 0; + + void *pIter = taosHashIterate(mgmt->ctxHash, NULL); + while (pIter) { + SQWTaskCtx *ctx = (SQWTaskCtx *)pIter; + void *key = taosHashGetKey(pIter, NULL); + QW_GET_QTID(key, qId, tId, eId); + + QW_LOCK(QW_WRITE, &ctx->lock); + + sId = ctx->sId; + + QW_TASK_DLOG_E("start to force stop task"); + + if (QW_EVENT_RECEIVED(ctx, QW_EVENT_DROP) || QW_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) { + QW_TASK_WLOG_E("task already dropping"); + QW_UNLOCK(QW_WRITE, &ctx->lock); + + pIter = taosHashIterate(mgmt->ctxHash, pIter); + continue; + } + + if (QW_QUERY_RUNNING(ctx)) { + qwKillTaskHandle(ctx, TSDB_CODE_VND_STOPPED); + QW_TASK_DLOG_E("task running, async killed"); + } else if (QW_FETCH_RUNNING(ctx)) { + QW_UPDATE_RSP_CODE(ctx, TSDB_CODE_VND_STOPPED); + QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_DROP); + QW_TASK_DLOG_E("task fetching, update drop received"); + } else { + qwDropTask(QW_FPARAMS()); + } + + QW_UNLOCK(QW_WRITE, &ctx->lock); + + pIter = taosHashIterate(mgmt->ctxHash, pIter); + } + + return TSDB_CODE_SUCCESS; +} + int32_t qwProcessHbLinkBroken(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { int32_t code = 0; SSchedulerHbRsp rsp = {0}; @@ -202,7 +246,7 @@ _return: } bool qwTaskNotInExec(SQWTaskCtx *ctx) { - qTaskInfo_t taskHandle = ctx->taskHandle; + qTaskInfo_t taskHandle = ctx->taskHandle; if (NULL == taskHandle || !qTaskIsExecuting(taskHandle)) { return true; } @@ -758,7 +802,8 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { } QW_LOCK(QW_WRITE, &ctx->lock); - if (atomic_load_8((int8_t*)&ctx->queryEnd) || (queryStop && (0 == atomic_load_8((int8_t *)&ctx->queryContinue))) || code) { + if (atomic_load_8((int8_t *)&ctx->queryEnd) || (queryStop && (0 == atomic_load_8((int8_t *)&ctx->queryContinue))) || + code) { // Note: query is not running anymore QW_SET_PHASE(ctx, QW_PHASE_POST_CQUERY); QW_UNLOCK(QW_WRITE, &ctx->lock); @@ -975,6 +1020,10 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) { qwDbgDumpMgmtInfo(mgmt); + if (gQWDebug.forceStop) { + (void)qwStopAllTasks(mgmt); + } + QW_LOCK(QW_READ, &mgmt->schLock); int32_t schNum = taosHashGetSize(mgmt->schHash); @@ -1188,45 +1237,9 @@ void qWorkerStopAllTasks(void *qWorkerMgmt) { QW_DLOG("start to stop all tasks, taskNum:%d", taosHashGetSize(mgmt->ctxHash)); - uint64_t qId, tId, sId; - int32_t eId; - int64_t rId = 0; - atomic_store_8(&mgmt->nodeStopped, 1); - void *pIter = taosHashIterate(mgmt->ctxHash, NULL); - while (pIter) { - SQWTaskCtx *ctx = (SQWTaskCtx *)pIter; - void *key = taosHashGetKey(pIter, NULL); - QW_GET_QTID(key, qId, tId, eId); - - QW_LOCK(QW_WRITE, &ctx->lock); - - sId = ctx->sId; - - QW_TASK_DLOG_E("start to force stop task"); - - if (QW_EVENT_RECEIVED(ctx, QW_EVENT_DROP) || QW_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) { - QW_TASK_WLOG_E("task already dropping"); - QW_UNLOCK(QW_WRITE, &ctx->lock); - - pIter = taosHashIterate(mgmt->ctxHash, pIter); - continue; - } - - if (QW_QUERY_RUNNING(ctx)) { - qwKillTaskHandle(ctx, TSDB_CODE_VND_STOPPED); - } else if (QW_FETCH_RUNNING(ctx)) { - QW_UPDATE_RSP_CODE(ctx, TSDB_CODE_VND_STOPPED); - QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_DROP); - } else { - (void)qwDropTask(QW_FPARAMS()); - } - - QW_UNLOCK(QW_WRITE, &ctx->lock); - - pIter = taosHashIterate(mgmt->ctxHash, pIter); - } + (void)qwStopAllTasks(mgmt); } void qWorkerDestroy(void **qWorkerMgmt) { diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index c1e958b055..25938a7678 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -69,7 +69,7 @@ int32_t sclConvertValueToSclParam(SValueNode *pValueNode, SScalarParam *out, int return code; } - colDataAppend(in.columnData, 0, nodesGetValueFromNode(pValueNode), false); + colDataSetVal(in.columnData, 0, nodesGetValueFromNode(pValueNode), false); colInfoDataEnsureCapacity(out->columnData, 1, true); code = vectorConvertSingleColImpl(&in, out, overflow, -1, -1); @@ -337,9 +337,9 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t SCL_RET(TSDB_CODE_OUT_OF_MEMORY); } if (TSDB_DATA_TYPE_NULL == valueNode->node.resType.type || valueNode->isNull) { - colDataAppendNULL(param->columnData, 0); + colDataSetNULL(param->columnData, 0); } else { - colDataAppend(param->columnData, 0, nodesGetValueFromNode(valueNode), false); + colDataSetVal(param->columnData, 0, nodesGetValueFromNode(valueNode), false); } break; } @@ -605,7 +605,7 @@ int32_t sclWalkCaseWhenList(SScalarCtx *ctx, SNodeList *pList, struct SListCell if (*equal) { bool isNull = colDataIsNull_s(pThen->columnData, (pThen->numOfRows > 1 ? rowIdx : 0)); char *pData = isNull ? NULL : colDataGetData(pThen->columnData, (pThen->numOfRows > 1 ? rowIdx : 0)); - colDataAppend(output->columnData, rowIdx, pData, isNull); + colDataSetVal(output->columnData, rowIdx, pData, isNull); if (0 == rowIdx && 1 == pCase->numOfRows && 1 == pWhen->numOfRows && 1 == pThen->numOfRows && totalRows > 1) { SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); @@ -619,7 +619,7 @@ int32_t sclWalkCaseWhenList(SScalarCtx *ctx, SNodeList *pList, struct SListCell if (pElse) { bool isNull = colDataIsNull_s(pElse->columnData, (pElse->numOfRows > 1 ? rowIdx : 0)); char *pData = isNull ? NULL : colDataGetData(pElse->columnData, (pElse->numOfRows > 1 ? rowIdx : 0)); - colDataAppend(output->columnData, rowIdx, pData, isNull); + colDataSetVal(output->columnData, rowIdx, pData, isNull); if (0 == rowIdx && 1 == pCase->numOfRows && 1 == pElse->numOfRows && totalRows > 1) { SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); @@ -629,7 +629,7 @@ int32_t sclWalkCaseWhenList(SScalarCtx *ctx, SNodeList *pList, struct SListCell goto _return; } - colDataAppend(output->columnData, rowIdx, NULL, true); + colDataSetVal(output->columnData, rowIdx, NULL, true); if (0 == rowIdx && 1 == pCase->numOfRows && totalRows > 1) { SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); @@ -668,7 +668,7 @@ int32_t sclWalkWhenList(SScalarCtx *ctx, SNodeList *pList, struct SListCell *pCe if (*whenValue) { bool isNull = colDataIsNull_s(pThen->columnData, (pThen->numOfRows > 1 ? rowIdx : 0)); char *pData = isNull ? NULL : colDataGetData(pThen->columnData, (pThen->numOfRows > 1 ? rowIdx : 0)); - colDataAppend(output->columnData, rowIdx, pData, isNull); + colDataSetVal(output->columnData, rowIdx, pData, isNull); if (preSingle && 0 == rowIdx && 1 == pWhen->numOfRows && 1 == pThen->numOfRows && totalRows > 1) { SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); @@ -687,7 +687,7 @@ int32_t sclWalkWhenList(SScalarCtx *ctx, SNodeList *pList, struct SListCell *pCe if (pElse) { bool isNull = colDataIsNull_s(pElse->columnData, (pElse->numOfRows > 1 ? rowIdx : 0)); char *pData = isNull ? NULL : colDataGetData(pElse->columnData, (pElse->numOfRows > 1 ? rowIdx : 0)); - colDataAppend(output->columnData, rowIdx, pData, isNull); + colDataSetVal(output->columnData, rowIdx, pData, isNull); if (preSingle && 0 == rowIdx && 1 == pElse->numOfRows && totalRows > 1) { SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); @@ -697,7 +697,7 @@ int32_t sclWalkWhenList(SScalarCtx *ctx, SNodeList *pList, struct SListCell *pCe goto _return; } - colDataAppend(output->columnData, rowIdx, NULL, true); + colDataSetVal(output->columnData, rowIdx, NULL, true); if (preSingle && 0 == rowIdx && totalRows > 1) { SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); @@ -815,7 +815,7 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o } if (complete) { - colDataAppend(output->columnData, i, (char *)&value, false); + colDataSetVal(output->columnData, i, (char *)&value, false); if (value) { numOfQualified++; } @@ -920,7 +920,7 @@ int32_t sclExecCaseWhen(SCaseWhenNode *node, SScalarCtx *ctx, SScalarParam *outp for (int32_t i = 0; i < rowNum; ++i) { bool *equal = (bool *)colDataGetData(comp.columnData, (comp.numOfRows > 1 ? i : 0)); if (*equal) { - colDataAppend(output->columnData, i, colDataGetData(pThen->columnData, (pThen->numOfRows > 1 ? i : 0)), + colDataSetVal(output->columnData, i, colDataGetData(pThen->columnData, (pThen->numOfRows > 1 ? i : 0)), colDataIsNull_s(pThen->columnData, (pThen->numOfRows > 1 ? i : 0))); if (0 == i && 1 == pCase->numOfRows && 1 == pWhen->numOfRows && 1 == pThen->numOfRows && rowNum > 1) { SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); @@ -938,7 +938,7 @@ int32_t sclExecCaseWhen(SCaseWhenNode *node, SScalarCtx *ctx, SScalarParam *outp for (int32_t i = 0; i < rowNum; ++i) { bool *whenValue = (bool *)colDataGetData(pWhen->columnData, (pWhen->numOfRows > 1 ? i : 0)); if (*whenValue) { - colDataAppend(output->columnData, i, colDataGetData(pThen->columnData, (pThen->numOfRows > 1 ? i : 0)), + colDataSetVal(output->columnData, i, colDataGetData(pThen->columnData, (pThen->numOfRows > 1 ? i : 0)), colDataIsNull_s(pThen->columnData, (pThen->numOfRows > 1 ? i : 0))); if (0 == i && 1 == pWhen->numOfRows && 1 == pThen->numOfRows && rowNum > 1) { SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 411b9b7ab5..f4c3669574 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -43,7 +43,7 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu float *out = (float *)pOutputData->pData; for (int32_t i = 0; i < pInput->numOfRows; ++i) { if (colDataIsNull_s(pInputData, i)) { - colDataAppendNULL(pOutputData, i); + colDataSetNULL(pOutputData, i); continue; } out[i] = (in[i] >= 0) ? in[i] : -in[i]; @@ -56,7 +56,7 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu double *out = (double *)pOutputData->pData; for (int32_t i = 0; i < pInput->numOfRows; ++i) { if (colDataIsNull_s(pInputData, i)) { - colDataAppendNULL(pOutputData, i); + colDataSetNULL(pOutputData, i); continue; } out[i] = (in[i] >= 0) ? in[i] : -in[i]; @@ -69,7 +69,7 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu int8_t *out = (int8_t *)pOutputData->pData; for (int32_t i = 0; i < pInput->numOfRows; ++i) { if (colDataIsNull_s(pInputData, i)) { - colDataAppendNULL(pOutputData, i); + colDataSetNULL(pOutputData, i); continue; } out[i] = (in[i] >= 0) ? in[i] : -in[i]; @@ -82,7 +82,7 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu int16_t *out = (int16_t *)pOutputData->pData; for (int32_t i = 0; i < pInput->numOfRows; ++i) { if (colDataIsNull_s(pInputData, i)) { - colDataAppendNULL(pOutputData, i); + colDataSetNULL(pOutputData, i); continue; } out[i] = (in[i] >= 0) ? in[i] : -in[i]; @@ -95,7 +95,7 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu int32_t *out = (int32_t *)pOutputData->pData; for (int32_t i = 0; i < pInput->numOfRows; ++i) { if (colDataIsNull_s(pInputData, i)) { - colDataAppendNULL(pOutputData, i); + colDataSetNULL(pOutputData, i); continue; } out[i] = (in[i] >= 0) ? in[i] : -in[i]; @@ -108,7 +108,7 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu int64_t *out = (int64_t *)pOutputData->pData; for (int32_t i = 0; i < pInput->numOfRows; ++i) { if (colDataIsNull_s(pInputData, i)) { - colDataAppendNULL(pOutputData, i); + colDataSetNULL(pOutputData, i); continue; } out[i] = (in[i] >= 0) ? in[i] : -in[i]; @@ -118,7 +118,7 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu case TSDB_DATA_TYPE_NULL: { for (int32_t i = 0; i < pInput->numOfRows; ++i) { - colDataAppendNULL(pOutputData, i); + colDataSetNULL(pOutputData, i); } break; } @@ -144,12 +144,12 @@ static int32_t doScalarFunctionUnique(SScalarParam *pInput, int32_t inputNum, SS for (int32_t i = 0; i < pInput->numOfRows; ++i) { if (colDataIsNull_s(pInputData, i) || IS_NULL_TYPE(type)) { - colDataAppendNULL(pOutputData, i); + colDataSetNULL(pOutputData, i); continue; } double result = valFn(getValueFn(pInputData->pData, i)); if (isinf(result) || isnan(result)) { - colDataAppendNULL(pOutputData, i); + colDataSetNULL(pOutputData, i); } else { out[i] = result; } @@ -179,29 +179,29 @@ static int32_t doScalarFunctionUnique2(SScalarParam *pInput, int32_t inputNum, S if (pInput[0].numOfRows == pInput[1].numOfRows) { for (int32_t i = 0; i < numOfRows; ++i) { if (colDataIsNull_s(pInputData[0], i) || colDataIsNull_s(pInputData[1], i) || hasNullType) { - colDataAppendNULL(pOutputData, i); + colDataSetNULL(pOutputData, i); continue; } result = valFn(getValueFn[0](pInputData[0]->pData, i), getValueFn[1](pInputData[1]->pData, i)); if (isinf(result) || isnan(result)) { - colDataAppendNULL(pOutputData, i); + colDataSetNULL(pOutputData, i); } else { out[i] = result; } } } else if (pInput[0].numOfRows == 1) { // left operand is constant if (colDataIsNull_s(pInputData[0], 0) || hasNullType) { - colDataAppendNNULL(pOutputData, 0, pInput[1].numOfRows); + colDataSetNNULL(pOutputData, 0, pInput[1].numOfRows); } else { for (int32_t i = 0; i < numOfRows; ++i) { if (colDataIsNull_s(pInputData[1], i)) { - colDataAppendNULL(pOutputData, i); + colDataSetNULL(pOutputData, i); continue; } result = valFn(getValueFn[0](pInputData[0]->pData, 0), getValueFn[1](pInputData[1]->pData, i)); if (isinf(result) || isnan(result)) { - colDataAppendNULL(pOutputData, i); + colDataSetNULL(pOutputData, i); continue; } @@ -210,17 +210,17 @@ static int32_t doScalarFunctionUnique2(SScalarParam *pInput, int32_t inputNum, S } } else if (pInput[1].numOfRows == 1) { if (colDataIsNull_s(pInputData[1], 0) || hasNullType) { - colDataAppendNNULL(pOutputData, 0, pInput[0].numOfRows); + colDataSetNNULL(pOutputData, 0, pInput[0].numOfRows); } else { for (int32_t i = 0; i < numOfRows; ++i) { if (colDataIsNull_s(pInputData[0], i)) { - colDataAppendNULL(pOutputData, i); + colDataSetNULL(pOutputData, i); continue; } result = valFn(getValueFn[0](pInputData[0]->pData, i), getValueFn[1](pInputData[1]->pData, 0)); if (isinf(result) || isnan(result)) { - colDataAppendNULL(pOutputData, i); + colDataSetNULL(pOutputData, i); continue; } @@ -247,7 +247,7 @@ static int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP for (int32_t i = 0; i < pInput->numOfRows; ++i) { if (colDataIsNull_s(pInputData, i)) { - colDataAppendNULL(pOutputData, i); + colDataSetNULL(pOutputData, i); continue; } out[i] = f1(in[i]); @@ -261,7 +261,7 @@ static int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP for (int32_t i = 0; i < pInput->numOfRows; ++i) { if (colDataIsNull_s(pInputData, i)) { - colDataAppendNULL(pOutputData, i); + colDataSetNULL(pOutputData, i); continue; } out[i] = d1(in[i]); @@ -271,7 +271,7 @@ static int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP case TSDB_DATA_TYPE_NULL: { for (int32_t i = 0; i < pInput->numOfRows; ++i) { - colDataAppendNULL(pOutputData, i); + colDataSetNULL(pOutputData, i); } break; } @@ -365,7 +365,7 @@ static int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarP for (int32_t i = 0; i < pInput->numOfRows; ++i) { if (colDataIsNull_s(pInputData, i)) { - colDataAppendNULL(pOutputData, i); + colDataSetNULL(pOutputData, i); continue; } @@ -447,7 +447,7 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu bool hasNull = false; for (int32_t i = 0; i < inputNum; ++i) { if (colDataIsNull_s(pInputData[i], k) || IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[i]))) { - colDataAppendNULL(pOutputData, k); + colDataSetNULL(pOutputData, k); hasNull = true; break; } @@ -468,7 +468,7 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu } } varDataSetLen(output, dataLen); - colDataAppend(pOutputData, k, output, false); + colDataSetVal(pOutputData, k, output, false); output += varDataTLen(output); } @@ -522,7 +522,7 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p for (int32_t k = 0; k < numOfRows; ++k) { if (colDataIsNull_s(pInputData[0], k) || IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[0]))) { - colDataAppendNULL(pOutputData, k); + colDataSetNULL(pOutputData, k); continue; } @@ -553,11 +553,11 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p } if (hasNull) { - colDataAppendNULL(pOutputData, k); + colDataSetNULL(pOutputData, k); memset(output, 0, dataLen); } else { varDataSetLen(output, dataLen); - colDataAppend(pOutputData, k, output, false); + colDataSetVal(pOutputData, k, output, false); output += varDataTLen(output); } } @@ -584,7 +584,7 @@ static int32_t doCaseConvFunction(SScalarParam *pInput, int32_t inputNum, SScala for (int32_t i = 0; i < pInput->numOfRows; ++i) { if (colDataIsNull_s(pInputData, i)) { - colDataAppendNULL(pOutputData, i); + colDataSetNULL(pOutputData, i); continue; } @@ -600,7 +600,7 @@ static int32_t doCaseConvFunction(SScalarParam *pInput, int32_t inputNum, SScala } } varDataSetLen(output, len); - colDataAppend(pOutputData, i, output, false); + colDataSetVal(pOutputData, i, output, false); output += varDataTLen(output); } @@ -622,7 +622,7 @@ static int32_t doTrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarPar for (int32_t i = 0; i < pInput->numOfRows; ++i) { if (colDataIsNull_s(pInputData, i)) { - colDataAppendNULL(pOutputData, i); + colDataSetNULL(pOutputData, i); continue; } @@ -631,7 +631,7 @@ static int32_t doTrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarPar int32_t charLen = (type == TSDB_DATA_TYPE_VARCHAR) ? len : len / TSDB_NCHAR_SIZE; trimFn(input, output, type, charLen); - colDataAppend(pOutputData, i, output, false); + colDataSetVal(pOutputData, i, output, false); output += varDataTLen(output); } @@ -660,7 +660,7 @@ int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu for (int32_t i = 0; i < pInput->numOfRows; ++i) { if (colDataIsNull_s(pInputData, i)) { - colDataAppendNULL(pOutputData, i); + colDataSetNULL(pOutputData, i); continue; } char *input = colDataGetData(pInput[0].columnData, i); @@ -682,7 +682,7 @@ int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu } varDataSetLen(output, resLen); - colDataAppend(pOutputData, i, output, false); + colDataSetVal(pOutputData, i, output, false); output += varDataTLen(output); } @@ -711,7 +711,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { if (colDataIsNull_s(pInput[0].columnData, i)) { - colDataAppendNULL(pOutput->columnData, i); + colDataSetNULL(pOutput->columnData, i); continue; } @@ -1014,7 +1014,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } } - colDataAppend(pOutput->columnData, i, output, false); + colDataSetVal(pOutput->columnData, i, output, false); } pOutput->numOfRows = pInput->numOfRows; @@ -1038,7 +1038,7 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam * for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { if (colDataIsNull_s(pInput[0].columnData, i)) { - colDataAppendNULL(pOutput->columnData, i); + colDataSetNULL(pOutput->columnData, i); continue; } @@ -1059,7 +1059,7 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam * } else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { timeVal = timeVal / (1000 * 1000 * 1000); } else { - colDataAppendNULL(pOutput->columnData, i); + colDataSetNULL(pOutput->columnData, i); continue; } hasFraction = true; @@ -1106,7 +1106,7 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam * memmove(buf + VARSTR_HEADER_SIZE, buf, len); varDataSetLen(buf, len); - colDataAppend(pOutput->columnData, i, buf, false); + colDataSetVal(pOutput->columnData, i, buf, false); } pOutput->numOfRows = pInput->numOfRows; @@ -1121,7 +1121,7 @@ int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarP for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { if (colDataIsNull_s(pInput[0].columnData, i)) { - colDataAppendNULL(pOutput->columnData, i); + colDataSetNULL(pOutput->columnData, i); continue; } char *input = colDataGetData(pInput[0].columnData, i); @@ -1129,9 +1129,9 @@ int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarP int64_t timeVal = 0; int32_t ret = convertStringToTimestamp(type, input, timePrec, &timeVal); if (ret != TSDB_CODE_SUCCESS) { - colDataAppendNULL(pOutput->columnData, i); + colDataSetNULL(pOutput->columnData, i); } else { - colDataAppend(pOutput->columnData, i, (char *)&timeVal, false); + colDataSetVal(pOutput->columnData, i, (char *)&timeVal, false); } } @@ -1163,7 +1163,7 @@ int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu } } - colDataAppend(pOutput->columnData, i, (const char *)pTag, false); + colDataSetVal(pOutput->columnData, i, (const char *)pTag, false); tTagFree(pTag); taosArrayDestroy(pTagVals); } @@ -1208,7 +1208,7 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { if (colDataIsNull_s(pInput[0].columnData, i)) { - colDataAppendNULL(pOutput->columnData, i); + colDataSetNULL(pOutput->columnData, i); continue; } @@ -1217,7 +1217,7 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara if (IS_VAR_DATA_TYPE(type)) { /* datetime format strings */ int32_t ret = convertStringToTimestamp(type, input, TSDB_TIME_PRECISION_NANO, &timeVal); if (ret != TSDB_CODE_SUCCESS) { - colDataAppendNULL(pOutput->columnData, i); + colDataSetNULL(pOutput->columnData, i); continue; } // If converted value is less than 10digits in second, use value in second instead @@ -1264,7 +1264,7 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara } else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) { timeVal = timeVal * factor; } else { - colDataAppendNULL(pOutput->columnData, i); + colDataSetNULL(pOutput->columnData, i); continue; } break; @@ -1279,7 +1279,7 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara } else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) { timeVal = timeVal * factor; } else { - colDataAppendNULL(pOutput->columnData, i); + colDataSetNULL(pOutput->columnData, i); continue; } break; @@ -1294,7 +1294,7 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara } else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) { timeVal = timeVal * factor / factor / 60 * 60 * factor; } else { - colDataAppendNULL(pOutput->columnData, i); + colDataSetNULL(pOutput->columnData, i); continue; } break; @@ -1309,7 +1309,7 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara } else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) { timeVal = timeVal * factor / factor / 3600 * 3600 * factor; } else { - colDataAppendNULL(pOutput->columnData, i); + colDataSetNULL(pOutput->columnData, i); continue; } break; @@ -1340,7 +1340,7 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara timeVal = timeVal * factor / factor / 86400 * 86400 * factor; } } else { - colDataAppendNULL(pOutput->columnData, i); + colDataSetNULL(pOutput->columnData, i); continue; } break; @@ -1355,7 +1355,7 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara } else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) { timeVal = timeVal * factor / factor / 604800 * 604800 * factor; } else { - colDataAppendNULL(pOutput->columnData, i); + colDataSetNULL(pOutput->columnData, i); continue; } break; @@ -1394,7 +1394,7 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara } } - colDataAppend(pOutput->columnData, i, (char *)&timeVal, false); + colDataSetVal(pOutput->columnData, i, (char *)&timeVal, false); } pOutput->numOfRows = pInput->numOfRows; @@ -1466,7 +1466,7 @@ int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p } if (hasNull) { - colDataAppendNULL(pOutput->columnData, i); + colDataSetNULL(pOutput->columnData, i); continue; } @@ -1528,7 +1528,7 @@ int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p } } - colDataAppend(pOutput->columnData, i, (char *)&result, false); + colDataSetVal(pOutput->columnData, i, (char *)&result, false); } pOutput->numOfRows = numOfRows; @@ -1542,7 +1542,7 @@ int32_t nowFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu int64_t ts = taosGetTimestamp(timePrec); for (int32_t i = 0; i < pInput->numOfRows; ++i) { - colDataAppendInt64(pOutput->columnData, i, &ts); + colDataSetInt64(pOutput->columnData, i, &ts); } pOutput->numOfRows = pInput->numOfRows; return TSDB_CODE_SUCCESS; @@ -1554,7 +1554,7 @@ int32_t todayFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOut int64_t ts = taosGetTimestampToday(timePrec); for (int32_t i = 0; i < pInput->numOfRows; ++i) { - colDataAppendInt64(pOutput->columnData, i, &ts); + colDataSetInt64(pOutput->columnData, i, &ts); } pOutput->numOfRows = pInput->numOfRows; return TSDB_CODE_SUCCESS; @@ -1565,7 +1565,7 @@ int32_t timezoneFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p memcpy(varDataVal(output), tsTimezoneStr, TD_TIMEZONE_LEN); varDataSetLen(output, strlen(tsTimezoneStr)); for (int32_t i = 0; i < pInput->numOfRows; ++i) { - colDataAppend(pOutput->columnData, i, output, false); + colDataSetVal(pOutput->columnData, i, output, false); } pOutput->numOfRows = pInput->numOfRows; return TSDB_CODE_SUCCESS; @@ -1728,34 +1728,34 @@ bool getTimePseudoFuncEnv(SFunctionNode *UNUSED_PARAM(pFunc), SFuncExecEnv *pEnv } int32_t qStartTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - colDataAppendInt64(pOutput->columnData, pOutput->numOfRows, (int64_t *)colDataGetData(pInput->columnData, 0)); + colDataSetInt64(pOutput->columnData, pOutput->numOfRows, (int64_t *)colDataGetData(pInput->columnData, 0)); return TSDB_CODE_SUCCESS; } int32_t qEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - colDataAppendInt64(pOutput->columnData, pOutput->numOfRows, (int64_t *)colDataGetData(pInput->columnData, 1)); + colDataSetInt64(pOutput->columnData, pOutput->numOfRows, (int64_t *)colDataGetData(pInput->columnData, 1)); return TSDB_CODE_SUCCESS; } int32_t winDurFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - colDataAppendInt64(pOutput->columnData, pOutput->numOfRows, (int64_t *)colDataGetData(pInput->columnData, 2)); + colDataSetInt64(pOutput->columnData, pOutput->numOfRows, (int64_t *)colDataGetData(pInput->columnData, 2)); return TSDB_CODE_SUCCESS; } int32_t winStartTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - colDataAppendInt64(pOutput->columnData, pOutput->numOfRows, (int64_t *)colDataGetData(pInput->columnData, 3)); + colDataSetInt64(pOutput->columnData, pOutput->numOfRows, (int64_t *)colDataGetData(pInput->columnData, 3)); return TSDB_CODE_SUCCESS; } int32_t winEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - colDataAppendInt64(pOutput->columnData, pOutput->numOfRows, (int64_t *)colDataGetData(pInput->columnData, 4)); + colDataSetInt64(pOutput->columnData, pOutput->numOfRows, (int64_t *)colDataGetData(pInput->columnData, 4)); return TSDB_CODE_SUCCESS; } int32_t qTbnameFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { char* p = colDataGetVarData(pInput->columnData, 0); - colDataAppendNItems(pOutput->columnData, pOutput->numOfRows, p, pInput->numOfRows); + colDataSetNItems(pOutput->columnData, pOutput->numOfRows, p, pInput->numOfRows); pOutput->numOfRows += pInput->numOfRows; return TSDB_CODE_SUCCESS; } @@ -1833,7 +1833,7 @@ int32_t sumScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam * } if (hasNull) { - colDataAppendNULL(pOutputData, 0); + colDataSetNULL(pOutputData, 0); } pOutput->numOfRows = 1; @@ -1945,7 +1945,7 @@ static int32_t doMinMaxScalarFunction(SScalarParam *pInput, int32_t inputNum, SS } if (hasNull) { - colDataAppendNULL(pOutputData, 0); + colDataSetNULL(pOutputData, 0); } pOutput->numOfRows = 1; @@ -2049,7 +2049,7 @@ int32_t avgScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam * } if (hasNull || (count == 0)) { - colDataAppendNULL(pOutputData, 0); + colDataSetNULL(pOutputData, 0); } else { if (IS_SIGNED_NUMERIC_TYPE(type)) { int64_t *out = (int64_t *)pOutputData->pData; @@ -2158,7 +2158,7 @@ int32_t stddevScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara double *out = (double *)pOutputData->pData; if (hasNull) { - colDataAppendNULL(pOutputData, 0); + colDataSetNULL(pOutputData, 0); } else { *out = 0; #if 0 @@ -2325,7 +2325,7 @@ int32_t leastSQRScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarPa } if (count == 0) { - colDataAppendNULL(pOutputData, 0); + colDataSetNULL(pOutputData, 0); } else { matrix[1][1] = (double)count; matrix[1][0] = matrix[0][1]; @@ -2341,7 +2341,7 @@ int32_t leastSQRScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarPa size_t len = snprintf(varDataVal(buf), sizeof(buf) - VARSTR_HEADER_SIZE, "{slop:%.6lf, intercept:%.6lf}", matrix02, matrix12); varDataSetLen(buf, len); - colDataAppend(pOutputData, 0, buf, false); + colDataSetVal(pOutputData, 0, buf, false); } pOutput->numOfRows = 1; @@ -2366,9 +2366,9 @@ int32_t percentileScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalar } if (hasNull) { - colDataAppendNULL(pOutputData, 0); + colDataSetNULL(pOutputData, 0); } else { - colDataAppend(pOutputData, 0, (char *)&val, false); + colDataSetVal(pOutputData, 0, (char *)&val, false); } pOutput->numOfRows = 1; @@ -2411,10 +2411,10 @@ int32_t spreadScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara } if (hasNull) { - colDataAppendNULL(pOutputData, 0); + colDataSetNULL(pOutputData, 0); } else { double result = max - min; - colDataAppend(pOutputData, 0, (char *)&result, false); + colDataSetVal(pOutputData, 0, (char *)&result, false); } pOutput->numOfRows = 1; @@ -2437,7 +2437,7 @@ int32_t nonCalcScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarPar double *out = (double *)pOutputData->pData; if (hasNull) { - colDataAppendNULL(pOutputData, 0); + colDataSetNULL(pOutputData, 0); } else { *out = 0; } @@ -2606,7 +2606,7 @@ int32_t stateCountScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalar for (int32_t i = 0; i < pInput->numOfRows; ++i) { if (colDataIsNull_s(pInputData, i)) { - colDataAppendNULL(pOutputData, i); + colDataSetNULL(pOutputData, i); continue; } @@ -2617,7 +2617,7 @@ int32_t stateCountScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalar } else { count = 0; } - colDataAppend(pOutputData, i, (char *)&out, false); + colDataSetVal(pOutputData, i, (char *)&out, false); } pOutput->numOfRows = pInput->numOfRows; @@ -2632,7 +2632,7 @@ int32_t stateDurationScalarFunction(SScalarParam *pInput, int32_t inputNum, SSca for (int32_t i = 0; i < pInput->numOfRows; ++i) { if (colDataIsNull_s(pInputData, i)) { - colDataAppendNULL(pOutputData, i); + colDataSetNULL(pOutputData, i); continue; } @@ -2641,7 +2641,7 @@ int32_t stateDurationScalarFunction(SScalarParam *pInput, int32_t inputNum, SSca if (ret) { out = 0; } - colDataAppend(pOutputData, i, (char *)&out, false); + colDataSetVal(pOutputData, i, (char *)&out, false); } pOutput->numOfRows = pInput->numOfRows; @@ -2882,7 +2882,7 @@ int32_t histogramScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP bins[k].upper, bins[k].percentage); } varDataSetLen(buf, len); - colDataAppend(pOutputData, k, buf, false); + colDataSetVal(pOutputData, k, buf, false); } taosMemoryFree(bins); @@ -2898,12 +2898,12 @@ int32_t selectScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara for (int32_t i = 0; i < pInput->numOfRows; ++i) { if (colDataIsNull_s(pInputData, i)) { - colDataAppendNULL(pOutputData, 0); + colDataSetNULL(pOutputData, 0); continue; } char *data = colDataGetData(pInputData, i); - colDataAppend(pOutputData, i, data, false); + colDataSetVal(pOutputData, i, data, false); } pOutput->numOfRows = 1; diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index e9f345215e..3fe016d444 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -242,7 +242,7 @@ static FORCE_INLINE void varToTimestamp(char *buf, SScalarParam *pOut, int32_t r value = 0; } - colDataAppendInt64(pOut->columnData, rowIndex, &value); + colDataSetInt64(pOut->columnData, rowIndex, &value); } static FORCE_INLINE void varToSigned(char *buf, SScalarParam *pOut, int32_t rowIndex, int32_t *overflow) { @@ -265,22 +265,22 @@ static FORCE_INLINE void varToSigned(char *buf, SScalarParam *pOut, int32_t rowI case TSDB_DATA_TYPE_TINYINT: { int8_t value = (int8_t)taosStr2Int8(buf, NULL, 10); - colDataAppendInt8(pOut->columnData, rowIndex, (int8_t *)&value); + colDataSetInt8(pOut->columnData, rowIndex, (int8_t *)&value); break; } case TSDB_DATA_TYPE_SMALLINT: { int16_t value = (int16_t)taosStr2Int16(buf, NULL, 10); - colDataAppendInt16(pOut->columnData, rowIndex, (int16_t *)&value); + colDataSetInt16(pOut->columnData, rowIndex, (int16_t *)&value); break; } case TSDB_DATA_TYPE_INT: { int32_t value = (int32_t)taosStr2Int32(buf, NULL, 10); - colDataAppendInt32(pOut->columnData, rowIndex, (int32_t *)&value); + colDataSetInt32(pOut->columnData, rowIndex, (int32_t *)&value); break; } case TSDB_DATA_TYPE_BIGINT: { int64_t value = (int64_t)taosStr2Int64(buf, NULL, 10); - colDataAppendInt64(pOut->columnData, rowIndex, (int64_t *)&value); + colDataSetInt64(pOut->columnData, rowIndex, (int64_t *)&value); break; } } @@ -305,22 +305,22 @@ static FORCE_INLINE void varToUnsigned(char *buf, SScalarParam *pOut, int32_t ro switch (pOut->columnData->info.type) { case TSDB_DATA_TYPE_UTINYINT: { uint8_t value = (uint8_t)taosStr2UInt8(buf, NULL, 10); - colDataAppendInt8(pOut->columnData, rowIndex, (int8_t *)&value); + colDataSetInt8(pOut->columnData, rowIndex, (int8_t *)&value); break; } case TSDB_DATA_TYPE_USMALLINT: { uint16_t value = (uint16_t)taosStr2UInt16(buf, NULL, 10); - colDataAppendInt16(pOut->columnData, rowIndex, (int16_t *)&value); + colDataSetInt16(pOut->columnData, rowIndex, (int16_t *)&value); break; } case TSDB_DATA_TYPE_UINT: { uint32_t value = (uint32_t)taosStr2UInt32(buf, NULL, 10); - colDataAppendInt32(pOut->columnData, rowIndex, (int32_t *)&value); + colDataSetInt32(pOut->columnData, rowIndex, (int32_t *)&value); break; } case TSDB_DATA_TYPE_UBIGINT: { uint64_t value = (uint64_t)taosStr2UInt64(buf, NULL, 10); - colDataAppendInt64(pOut->columnData, rowIndex, (int64_t *)&value); + colDataSetInt64(pOut->columnData, rowIndex, (int64_t *)&value); break; } } @@ -329,18 +329,18 @@ static FORCE_INLINE void varToUnsigned(char *buf, SScalarParam *pOut, int32_t ro static FORCE_INLINE void varToFloat(char *buf, SScalarParam *pOut, int32_t rowIndex, int32_t *overflow) { if (TSDB_DATA_TYPE_FLOAT == pOut->columnData->info.type) { float value = taosStr2Float(buf, NULL); - colDataAppendFloat(pOut->columnData, rowIndex, &value); + colDataSetFloat(pOut->columnData, rowIndex, &value); return; } double value = taosStr2Double(buf, NULL); - colDataAppendDouble(pOut->columnData, rowIndex, &value); + colDataSetDouble(pOut->columnData, rowIndex, &value); } static FORCE_INLINE void varToBool(char *buf, SScalarParam *pOut, int32_t rowIndex, int32_t *overflow) { int64_t value = taosStr2Int64(buf, NULL, 10); bool v = (value != 0) ? true : false; - colDataAppendInt8(pOut->columnData, rowIndex, (int8_t *)&v); + colDataSetInt8(pOut->columnData, rowIndex, (int8_t *)&v); } // todo remove this malloc @@ -357,7 +357,7 @@ static FORCE_INLINE void varToNchar(char *buf, SScalarParam *pOut, int32_t rowIn } varDataSetLen(t, len); - colDataAppend(pOut->columnData, rowIndex, t, false); + colDataSetVal(pOut->columnData, rowIndex, t, false); taosMemoryFree(t); } @@ -372,7 +372,7 @@ static FORCE_INLINE void ncharToVar(char *buf, SScalarParam *pOut, int32_t rowIn } varDataSetLen(t, len); - colDataAppend(pOut->columnData, rowIndex, t, false); + colDataSetVal(pOut->columnData, rowIndex, t, false); taosMemoryFree(t); } @@ -409,7 +409,7 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) { for (int32_t i = pCtx->startIndex; i <= pCtx->endIndex; ++i) { if (IS_HELPER_NULL(pCtx->pIn->columnData, i)) { - colDataAppendNULL(pCtx->pOut->columnData, i); + colDataSetNULL(pCtx->pOut->columnData, i); continue; } @@ -599,7 +599,7 @@ int32_t vectorConvertToVarData(SSclVectorConvCtx *pCtx) { pCtx->inType == TSDB_DATA_TYPE_TIMESTAMP) { for (int32_t i = pCtx->startIndex; i <= pCtx->endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; } @@ -610,13 +610,13 @@ int32_t vectorConvertToVarData(SSclVectorConvCtx *pCtx) { if (pCtx->outType == TSDB_DATA_TYPE_NCHAR) { varToNchar(tmp, pCtx->pOut, i, NULL); } else { - colDataAppend(pOutputCol, i, (char *)tmp, false); + colDataSetVal(pOutputCol, i, (char *)tmp, false); } } } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inType)) { for (int32_t i = pCtx->startIndex; i <= pCtx->endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; } @@ -627,13 +627,13 @@ int32_t vectorConvertToVarData(SSclVectorConvCtx *pCtx) { if (pCtx->outType == TSDB_DATA_TYPE_NCHAR) { varToNchar(tmp, pCtx->pOut, i, NULL); } else { - colDataAppend(pOutputCol, i, (char *)tmp, false); + colDataSetVal(pOutputCol, i, (char *)tmp, false); } } } else if (IS_FLOAT_TYPE(pCtx->inType)) { for (int32_t i = pCtx->startIndex; i <= pCtx->endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; } @@ -644,7 +644,7 @@ int32_t vectorConvertToVarData(SSclVectorConvCtx *pCtx) { if (pCtx->outType == TSDB_DATA_TYPE_NCHAR) { varToNchar(tmp, pCtx->pOut, i, NULL); } else { - colDataAppend(pOutputCol, i, (char *)tmp, false); + colDataSetVal(pOutputCol, i, (char *)tmp, false); } } } else { @@ -722,52 +722,52 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, case TSDB_DATA_TYPE_BOOL: { for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; } bool value = 0; GET_TYPED_DATA(value, bool, cCtx.inType, colDataGetData(pInputCol, i)); - colDataAppendInt8(pOutputCol, i, (int8_t *)&value); + colDataSetInt8(pOutputCol, i, (int8_t *)&value); } break; } case TSDB_DATA_TYPE_TINYINT: { for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; } int8_t value = 0; GET_TYPED_DATA(value, int8_t, cCtx.inType, colDataGetData(pInputCol, i)); - colDataAppendInt8(pOutputCol, i, (int8_t *)&value); + colDataSetInt8(pOutputCol, i, (int8_t *)&value); } break; } case TSDB_DATA_TYPE_SMALLINT: { for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; } int16_t value = 0; GET_TYPED_DATA(value, int16_t, cCtx.inType, colDataGetData(pInputCol, i)); - colDataAppendInt16(pOutputCol, i, (int16_t *)&value); + colDataSetInt16(pOutputCol, i, (int16_t *)&value); } break; } case TSDB_DATA_TYPE_INT: { for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; } int32_t value = 0; GET_TYPED_DATA(value, int32_t, cCtx.inType, colDataGetData(pInputCol, i)); - colDataAppendInt32(pOutputCol, i, (int32_t *)&value); + colDataSetInt32(pOutputCol, i, (int32_t *)&value); } break; } @@ -775,91 +775,91 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, case TSDB_DATA_TYPE_TIMESTAMP: { for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; } int64_t value = 0; GET_TYPED_DATA(value, int64_t, cCtx.inType, colDataGetData(pInputCol, i)); - colDataAppendInt64(pOutputCol, i, (int64_t *)&value); + colDataSetInt64(pOutputCol, i, (int64_t *)&value); } break; } case TSDB_DATA_TYPE_UTINYINT: { for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; } uint8_t value = 0; GET_TYPED_DATA(value, uint8_t, cCtx.inType, colDataGetData(pInputCol, i)); - colDataAppendInt8(pOutputCol, i, (int8_t *)&value); + colDataSetInt8(pOutputCol, i, (int8_t *)&value); } break; } case TSDB_DATA_TYPE_USMALLINT: { for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; } uint16_t value = 0; GET_TYPED_DATA(value, uint16_t, cCtx.inType, colDataGetData(pInputCol, i)); - colDataAppendInt16(pOutputCol, i, (int16_t *)&value); + colDataSetInt16(pOutputCol, i, (int16_t *)&value); } break; } case TSDB_DATA_TYPE_UINT: { for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; } uint32_t value = 0; GET_TYPED_DATA(value, uint32_t, cCtx.inType, colDataGetData(pInputCol, i)); - colDataAppendInt32(pOutputCol, i, (int32_t *)&value); + colDataSetInt32(pOutputCol, i, (int32_t *)&value); } break; } case TSDB_DATA_TYPE_UBIGINT: { for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; } uint64_t value = 0; GET_TYPED_DATA(value, uint64_t, cCtx.inType, colDataGetData(pInputCol, i)); - colDataAppendInt64(pOutputCol, i, (int64_t *)&value); + colDataSetInt64(pOutputCol, i, (int64_t *)&value); } break; } case TSDB_DATA_TYPE_FLOAT: { for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; } float value = 0; GET_TYPED_DATA(value, float, cCtx.inType, colDataGetData(pInputCol, i)); - colDataAppendFloat(pOutputCol, i, (float *)&value); + colDataSetFloat(pOutputCol, i, (float *)&value); } break; } case TSDB_DATA_TYPE_DOUBLE: { for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; } double value = 0; GET_TYPED_DATA(value, double, cCtx.inType, colDataGetData(pInputCol, i)); - colDataAppendDouble(pOutputCol, i, (double *)&value); + colDataSetDouble(pOutputCol, i, (double *)&value); } break; } @@ -1007,11 +1007,11 @@ static void vectorMathAddHelper(SColumnInfoData *pLeftCol, SColumnInfoData *pRig double *output = (double *)pOutputCol->pData; if (IS_HELPER_NULL(pRightCol, 0)) { // Set pLeft->numOfRows NULL value - colDataAppendNNULL(pOutputCol, 0, numOfRows); + colDataSetNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { if (IS_HELPER_NULL(pLeftCol, i)) { - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; // TODO set null or ignore } *output = getVectorDoubleValueFnLeft(LEFT_COL, i) + getVectorDoubleValueFnRight(RIGHT_COL, 0); @@ -1027,11 +1027,11 @@ static void vectorMathTsAddHelper(SColumnInfoData *pLeftCol, SColumnInfoData *pR int64_t *output = (int64_t *)pOutputCol->pData; if (IS_HELPER_NULL(pRightCol, 0)) { // Set pLeft->numOfRows NULL value - colDataAppendNNULL(pOutputCol, 0, numOfRows); + colDataSetNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { if (IS_HELPER_NULL(pLeftCol, i)) { - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; // TODO set null or ignore } *output = @@ -1103,7 +1103,7 @@ void vectorMathAdd(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut } else if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { if (IS_NULL) { - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; // TODO set null or ignore } *output = getVectorBigintValueFnLeft(pLeftCol->pData, i) + getVectorBigintValueFnRight(pRightCol->pData, i); @@ -1117,7 +1117,7 @@ void vectorMathAdd(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { if (IS_NULL) { - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; // TODO set null or ignore } *output = getVectorDoubleValueFnLeft(LEFT_COL, i) + getVectorDoubleValueFnRight(RIGHT_COL, i); @@ -1142,11 +1142,11 @@ static void vectorMathSubHelper(SColumnInfoData *pLeftCol, SColumnInfoData *pRig double *output = (double *)pOutputCol->pData; if (IS_HELPER_NULL(pRightCol, 0)) { // Set pLeft->numOfRows NULL value - colDataAppendNNULL(pOutputCol, 0, numOfRows); + colDataSetNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { if (IS_HELPER_NULL(pLeftCol, i)) { - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; // TODO set null or ignore } *output = (getVectorDoubleValueFnLeft(LEFT_COL, i) - getVectorDoubleValueFnRight(RIGHT_COL, 0)) * factor; @@ -1162,11 +1162,11 @@ static void vectorMathTsSubHelper(SColumnInfoData *pLeftCol, SColumnInfoData *pR int64_t *output = (int64_t *)pOutputCol->pData; if (IS_HELPER_NULL(pRightCol, 0)) { // Set pLeft->numOfRows NULL value - colDataAppendNNULL(pOutputCol, 0, numOfRows); + colDataSetNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { if (IS_HELPER_NULL(pLeftCol, i)) { - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; // TODO set null or ignore } *output = @@ -1204,7 +1204,7 @@ void vectorMathSub(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut } else if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { if (IS_NULL) { - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; // TODO set null or ignore } *output = getVectorBigintValueFnLeft(pLeftCol->pData, i) - getVectorBigintValueFnRight(pRightCol->pData, i); @@ -1218,7 +1218,7 @@ void vectorMathSub(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { if (IS_NULL) { - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; // TODO set null or ignore } *output = getVectorDoubleValueFnLeft(LEFT_COL, i) - getVectorDoubleValueFnRight(RIGHT_COL, i); @@ -1243,11 +1243,11 @@ static void vectorMathMultiplyHelper(SColumnInfoData *pLeftCol, SColumnInfoData double *output = (double *)pOutputCol->pData; if (IS_HELPER_NULL(pRightCol, 0)) { // Set pLeft->numOfRows NULL value - colDataAppendNNULL(pOutputCol, 0, numOfRows); + colDataSetNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { if (IS_HELPER_NULL(pLeftCol, i)) { - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; // TODO set null or ignore } *output = getVectorDoubleValueFnLeft(LEFT_COL, i) * getVectorDoubleValueFnRight(RIGHT_COL, 0); @@ -1273,7 +1273,7 @@ void vectorMathMultiply(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { if (IS_NULL) { - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; // TODO set null or ignore } *output = getVectorDoubleValueFnLeft(LEFT_COL, i) * getVectorDoubleValueFnRight(RIGHT_COL, i); @@ -1306,18 +1306,18 @@ void vectorMathDivide(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *p if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { if (IS_NULL || (getVectorDoubleValueFnRight(RIGHT_COL, i) == 0)) { // divide by 0 check - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; } *output = getVectorDoubleValueFnLeft(LEFT_COL, i) / getVectorDoubleValueFnRight(RIGHT_COL, i); } } else if (pLeft->numOfRows == 1) { if (IS_HELPER_NULL(pLeftCol, 0)) { // Set pLeft->numOfRows NULL value - colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows); + colDataSetNNULL(pOutputCol, 0, pRight->numOfRows); } else { for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) { if (IS_HELPER_NULL(pRightCol, i) || (getVectorDoubleValueFnRight(RIGHT_COL, i) == 0)) { // divide by 0 check - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; } *output = getVectorDoubleValueFnLeft(LEFT_COL, 0) / getVectorDoubleValueFnRight(RIGHT_COL, i); @@ -1326,11 +1326,11 @@ void vectorMathDivide(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *p } else if (pRight->numOfRows == 1) { if (IS_HELPER_NULL(pRightCol, 0) || (getVectorDoubleValueFnRight(RIGHT_COL, 0) == 0)) { // Set pLeft->numOfRows NULL value (divde by 0 check) - colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows); + colDataSetNNULL(pOutputCol, 0, pLeft->numOfRows); } else { for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) { if (IS_HELPER_NULL(pLeftCol, i)) { - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; } *output = getVectorDoubleValueFnLeft(LEFT_COL, i) / getVectorDoubleValueFnRight(RIGHT_COL, 0); @@ -1361,14 +1361,14 @@ void vectorMathRemainder(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { if (IS_NULL) { - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; } double lx = getVectorDoubleValueFnLeft(LEFT_COL, i); double rx = getVectorDoubleValueFnRight(RIGHT_COL, i); if (isnan(lx) || isinf(lx) || isnan(rx) || isinf(rx) || FLT_EQUAL(rx, 0)) { - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; } @@ -1377,17 +1377,17 @@ void vectorMathRemainder(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam } else if (pLeft->numOfRows == 1) { double lx = getVectorDoubleValueFnLeft(LEFT_COL, 0); if (IS_HELPER_NULL(pLeftCol, 0)) { // Set pLeft->numOfRows NULL value - colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows); + colDataSetNNULL(pOutputCol, 0, pRight->numOfRows); } else { for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) { if (IS_HELPER_NULL(pRightCol, i)) { - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; } double rx = getVectorDoubleValueFnRight(RIGHT_COL, i); if (isnan(rx) || isinf(rx) || FLT_EQUAL(rx, 0)) { - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; } @@ -1397,17 +1397,17 @@ void vectorMathRemainder(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam } else if (pRight->numOfRows == 1) { double rx = getVectorDoubleValueFnRight(RIGHT_COL, 0); if (IS_HELPER_NULL(pRightCol, 0) || FLT_EQUAL(rx, 0)) { // Set pLeft->numOfRows NULL value - colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows); + colDataSetNNULL(pOutputCol, 0, pLeft->numOfRows); } else { for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) { if (IS_HELPER_NULL(pLeftCol, i)) { - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; } double lx = getVectorDoubleValueFnLeft(LEFT_COL, i); if (isnan(lx) || isinf(lx)) { - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; } @@ -1436,7 +1436,7 @@ void vectorMathMinus(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pO double *output = (double *)pOutputCol->pData; for (; i < pLeft->numOfRows && i >= 0; i += step, output += 1) { if (IS_HELPER_NULL(pLeftCol, i)) { - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; } double result = getVectorDoubleValueFnLeft(LEFT_COL, i); @@ -1451,11 +1451,11 @@ void vectorAssign(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, pOut->numOfRows = pLeft->numOfRows; if (colDataIsNull_s(pRight->columnData, 0)) { - colDataAppendNNULL(pOutputCol, 0, pOut->numOfRows); + colDataSetNNULL(pOutputCol, 0, pOut->numOfRows); } else { char *d = colDataGetData(pRight->columnData, 0); for (int32_t i = 0; i < pOut->numOfRows; ++i) { - colDataAppend(pOutputCol, i, d, false); + colDataSetVal(pOutputCol, i, d, false); } } @@ -1471,11 +1471,11 @@ static void vectorBitAndHelper(SColumnInfoData *pLeftCol, SColumnInfoData *pRigh int64_t *output = (int64_t *)pOutputCol->pData; if (IS_HELPER_NULL(pRightCol, 0)) { // Set pLeft->numOfRows NULL value - colDataAppendNNULL(pOutputCol, 0, numOfRows); + colDataSetNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { if (IS_HELPER_NULL(pLeftCol, i)) { - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; // TODO set null or ignore } *output = getVectorBigintValueFnLeft(LEFT_COL, i) & getVectorBigintValueFnRight(RIGHT_COL, 0); @@ -1501,7 +1501,7 @@ void vectorBitAnd(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { if (IS_NULL) { - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; // TODO set null or ignore } *output = getVectorBigintValueFnLeft(LEFT_COL, i) & getVectorBigintValueFnRight(RIGHT_COL, i); @@ -1524,12 +1524,12 @@ static void vectorBitOrHelper(SColumnInfoData *pLeftCol, SColumnInfoData *pRight int64_t *output = (int64_t *)pOutputCol->pData; if (IS_HELPER_NULL(pRightCol, 0)) { // Set pLeft->numOfRows NULL value - colDataAppendNNULL(pOutputCol, 0, numOfRows); + colDataSetNNULL(pOutputCol, 0, numOfRows); } else { int64_t rx = getVectorBigintValueFnRight(RIGHT_COL, 0); for (; i >= 0 && i < numOfRows; i += step, output += 1) { if (IS_HELPER_NULL(pLeftCol, i)) { - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; // TODO set null or ignore } *output = getVectorBigintValueFnLeft(LEFT_COL, i) | rx; @@ -1555,7 +1555,7 @@ void vectorBitOr(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { if (IS_NULL) { - colDataAppendNULL(pOutputCol, i); + colDataSetNULL(pOutputCol, i); continue; // TODO set null or ignore } *output = getVectorBigintValueFnLeft(LEFT_COL, i) | getVectorBigintValueFnRight(RIGHT_COL, i); @@ -1617,7 +1617,7 @@ int32_t doVectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarPa if (IS_HELPER_NULL(pLeft->columnData, leftIndex) || IS_HELPER_NULL(pRight->columnData, rightIndex)) { bool res = false; - colDataAppendInt8(pOut->columnData, i, (int8_t *)&res); + colDataSetInt8(pOut->columnData, i, (int8_t *)&res); continue; } @@ -1640,10 +1640,10 @@ int32_t doVectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarPa } if (!result) { - colDataAppendInt8(pOut->columnData, i, (int8_t *)&result); + colDataSetInt8(pOut->columnData, i, (int8_t *)&result); } else { bool res = filterDoCompare(fp, optr, pLeftData, pRightData); - colDataAppendInt8(pOut->columnData, i, (int8_t *)&res); + colDataSetInt8(pOut->columnData, i, (int8_t *)&res); if (res) { ++num; } @@ -1690,13 +1690,13 @@ void doVectorCompare(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pO for (; i >= 0 && i < pLeft->numOfRows; i += step) { if (IS_HELPER_NULL(pLeft->columnData, i)) { bool res = false; - colDataAppendInt8(pOut->columnData, i, (int8_t *)&res); + colDataSetInt8(pOut->columnData, i, (int8_t *)&res); continue; } char *pLeftData = colDataGetData(pLeft->columnData, i); bool res = filterDoCompare(fp, optr, pLeftData, pRight->pHashFilter); - colDataAppendInt8(pOut->columnData, i, (int8_t *)&res); + colDataSetInt8(pOut->columnData, i, (int8_t *)&res); if (res) { pOut->numOfQualified++; } @@ -1783,7 +1783,7 @@ void vectorNotMatch(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOu void vectorIsNull(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t _ord) { for (int32_t i = 0; i < pLeft->numOfRows; ++i) { int8_t v = IS_HELPER_NULL(pLeft->columnData, i) ? 1 : 0; - colDataAppendInt8(pOut->columnData, i, &v); + colDataSetInt8(pOut->columnData, i, &v); } pOut->numOfRows = pLeft->numOfRows; } @@ -1791,7 +1791,7 @@ void vectorIsNull(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, void vectorNotNull(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t _ord) { for (int32_t i = 0; i < pLeft->numOfRows; ++i) { int8_t v = IS_HELPER_NULL(pLeft->columnData, i) ? 0 : 1; - colDataAppendInt8(pOut->columnData, i, &v); + colDataSetInt8(pOut->columnData, i, &v); } pOut->numOfRows = pLeft->numOfRows; } @@ -1801,7 +1801,7 @@ void vectorIsTrue(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, for (int32_t i = 0; i < pOut->numOfRows; ++i) { if (colDataIsNull_s(pOut->columnData, i)) { int8_t v = 0; - colDataAppendInt8(pOut->columnData, i, &v); + colDataSetInt8(pOut->columnData, i, &v); colDataClearNull_f(pOut->columnData->nullbitmap, i); } } @@ -1844,7 +1844,7 @@ void vectorJsonContains(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam getJsonValue(pLeftData, jsonKey, &isExist); } - colDataAppend(pOutputCol, i, (const char *)(&isExist), false); + colDataSetVal(pOutputCol, i, (const char *)(&isExist), false); } taosMemoryFree(jsonKey); } @@ -1870,7 +1870,7 @@ void vectorJsonArrow(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pO bool isExist = false; STagVal value = getJsonValue(pLeftData, jsonKey, &isExist); char *data = isExist ? tTagValToData(&value, true) : NULL; - colDataAppend(pOutputCol, i, data, data == NULL); + colDataSetVal(pOutputCol, i, data, data == NULL); if (isExist && IS_VAR_DATA_TYPE(value.type) && data) { taosMemoryFree(data); } diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index 2f518e2e59..b59e89fe0d 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -111,7 +111,7 @@ void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); for (int32_t i = 0; i < rowNum; ++i) { - colDataAppend(pColumn, i, (const char *)value, false); + colDataSetVal(pColumn, i, (const char *)value, false); if (IS_VAR_DATA_TYPE(dataType)) { value = (char *)value + varDataTLen(value); } else { @@ -135,7 +135,7 @@ void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); for (int32_t i = 0; i < rowNum; ++i) { - colDataAppend(pColumn, i, (const char *)value, false); + colDataSetVal(pColumn, i, (const char *)value, false); if (IS_VAR_DATA_TYPE(dataType)) { value = (char *)value + varDataTLen(value); } else { @@ -634,8 +634,8 @@ TEST(columnTest, binary_column_is_null) { flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); SColumnInfoData *pcolumn = (SColumnInfoData *)taosArrayGetLast(src->pDataBlock); - colDataAppend(pcolumn, 2, NULL, true); - colDataAppend(pcolumn, 4, NULL, true); + colDataSetVal(pcolumn, 2, NULL, true); + colDataSetVal(pcolumn, 4, NULL, true); flttMakeOpNode(&opNode, OP_TYPE_IS_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); SFilterInfo *filter = NULL; @@ -682,7 +682,7 @@ TEST(columnTest, binary_column_is_not_null) { flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); SColumnInfoData *pcolumn = (SColumnInfoData *)taosArrayGetLast(src->pDataBlock); - colDataAppend(pcolumn, 4, NULL, true); + colDataSetVal(pcolumn, 4, NULL, true); flttMakeOpNode(&opNode, OP_TYPE_IS_NOT_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 97002ed9bf..fe86e18ce3 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -158,7 +158,7 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in res->info.rows = rowNum; SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); for (int32_t i = 0; i < rowNum; ++i) { - colDataAppend(pColumn, i, (const char *)value, false); + colDataSetVal(pColumn, i, (const char *)value, false); if (IS_VAR_DATA_TYPE(dataType)) { value = (char *)value + varDataTLen(value); } else { @@ -183,7 +183,7 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); for (int32_t i = 0; i < rowNum; ++i) { - colDataAppend(pColumn, i, (const char *)value, false); + colDataSetVal(pColumn, i, (const char *)value, false); if (IS_VAR_DATA_TYPE(dataType)) { value = (char *)value + varDataTLen(value); } else { @@ -1880,8 +1880,8 @@ TEST(columnTest, binary_column_is_null) { scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); SColumnInfoData *pcolumn = (SColumnInfoData *)taosArrayGetLast(src->pDataBlock); - colDataAppend(pcolumn, 2, NULL, true); - colDataAppend(pcolumn, 4, NULL, true); + colDataSetVal(pcolumn, 2, NULL, true); + colDataSetVal(pcolumn, 4, NULL, true); scltMakeOpNode(&opNode, OP_TYPE_IS_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); @@ -1924,7 +1924,7 @@ TEST(columnTest, binary_column_is_not_null) { scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); SColumnInfoData *pcolumn = (SColumnInfoData *)taosArrayGetLast(src->pDataBlock); - colDataAppend(pcolumn, 4, NULL, true); + colDataSetVal(pcolumn, 4, NULL, true); scltMakeOpNode(&opNode, OP_TYPE_IS_NOT_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); @@ -2030,7 +2030,7 @@ void scltMakeDataBlock(SScalarParam **pInput, int32_t type, void *pVal, int32_t if (setVal) { for (int32_t i = 0; i < num; ++i) { - colDataAppend(input->columnData, i, (const char *)pVal, false); + colDataSetVal(input->columnData, i, (const char *)pVal, false); } } else { // memset(input->data, 0, num * bytes); @@ -2225,7 +2225,7 @@ TEST(ScalarFunctionTest, absFunction_column) { scltMakeDataBlock(&pOutput, type, 0, rowNum, false); for (int32_t i = 0; i < rowNum; ++i) { int8_t v = val_tinyint + i; - colDataAppend(pInput->columnData, i, (const char *)&v, false); + colDataSetVal(pInput->columnData, i, (const char *)&v, false); PRINTF("tiny_int before ABS:%d\n", v); } @@ -2243,7 +2243,7 @@ TEST(ScalarFunctionTest, absFunction_column) { scltMakeDataBlock(&pOutput, type, 0, rowNum, false); for (int32_t i = 0; i < rowNum; ++i) { int8_t v = val_tinyint + i; - colDataAppend(pInput->columnData, i, (const char *)&v, false); + colDataSetVal(pInput->columnData, i, (const char *)&v, false); PRINTF("tiny_int before ABS:%d\n", v); } @@ -2263,7 +2263,7 @@ TEST(ScalarFunctionTest, absFunction_column) { scltMakeDataBlock(&pOutput, type, 0, rowNum, false); for (int32_t i = 0; i < rowNum; ++i) { int16_t v = val_smallint + i; - colDataAppend(pInput->columnData, i, (const char *)&v, false); + colDataSetVal(pInput->columnData, i, (const char *)&v, false); PRINTF("small_int before ABS:%d\n", v); } @@ -2281,7 +2281,7 @@ TEST(ScalarFunctionTest, absFunction_column) { scltMakeDataBlock(&pOutput, type, 0, rowNum, false); for (int32_t i = 0; i < rowNum; ++i) { int16_t v = val_smallint + i; - colDataAppend(pInput->columnData, i, (const char *)&v, false); + colDataSetVal(pInput->columnData, i, (const char *)&v, false); PRINTF("small_int before ABS:%d\n", v); } @@ -2301,7 +2301,7 @@ TEST(ScalarFunctionTest, absFunction_column) { scltMakeDataBlock(&pOutput, type, 0, rowNum, false); for (int32_t i = 0; i < rowNum; ++i) { int32_t v = val_int + i; - colDataAppend(pInput->columnData, i, (const char *)&v, false); + colDataSetVal(pInput->columnData, i, (const char *)&v, false); PRINTF("int before ABS:%d\n", v); } @@ -2319,7 +2319,7 @@ TEST(ScalarFunctionTest, absFunction_column) { scltMakeDataBlock(&pOutput, type, 0, rowNum, false); for (int32_t i = 0; i < rowNum; ++i) { int32_t v = val_int + i; - colDataAppend(pInput->columnData, i, (const char *)&v, false); + colDataSetVal(pInput->columnData, i, (const char *)&v, false); PRINTF("int before ABS:%d\n", v); } @@ -2339,7 +2339,7 @@ TEST(ScalarFunctionTest, absFunction_column) { scltMakeDataBlock(&pOutput, type, 0, rowNum, false); for (int32_t i = 0; i < rowNum; ++i) { float v = val_float + i; - colDataAppend(pInput->columnData, i, (const char *)&v, false); + colDataSetVal(pInput->columnData, i, (const char *)&v, false); PRINTF("float before ABS:%f\n", v); } @@ -2357,7 +2357,7 @@ TEST(ScalarFunctionTest, absFunction_column) { scltMakeDataBlock(&pOutput, type, 0, rowNum, false); for (int32_t i = 0; i < rowNum; ++i) { float v = val_float + i; - colDataAppend(pInput->columnData, i, (const char *)&v, false); + colDataSetVal(pInput->columnData, i, (const char *)&v, false); PRINTF("float before ABS:%f\n", v); } @@ -2377,7 +2377,7 @@ TEST(ScalarFunctionTest, absFunction_column) { scltMakeDataBlock(&pOutput, type, 0, rowNum, false); for (int32_t i = 0; i < rowNum; ++i) { double v = val_double + i; - colDataAppend(pInput->columnData, i, (const char *)&v, false); + colDataSetVal(pInput->columnData, i, (const char *)&v, false); PRINTF("double before ABS:%f\n", v); } @@ -2395,7 +2395,7 @@ TEST(ScalarFunctionTest, absFunction_column) { scltMakeDataBlock(&pOutput, type, 0, rowNum, false); for (int32_t i = 0; i < rowNum; ++i) { double v = val_double + i; - colDataAppend(pInput->columnData, i, (const char *)&v, false); + colDataSetVal(pInput->columnData, i, (const char *)&v, false); PRINTF("double before ABS:%f\n", v); } @@ -2466,7 +2466,7 @@ TEST(ScalarFunctionTest, sinFunction_column) { scltMakeDataBlock(&pInput, type, 0, rowNum, false); scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); for (int32_t i = 0; i < rowNum; ++i) { - colDataAppend(pInput->columnData, i, (const char *)&val_tinyint[i], false); + colDataSetVal(pInput->columnData, i, (const char *)&val_tinyint[i], false); PRINTF("tiny_int before SIN:%d\n", *(int8_t *)colDataGetData(pInput->columnData, i)); } @@ -2485,7 +2485,7 @@ TEST(ScalarFunctionTest, sinFunction_column) { scltMakeDataBlock(&pInput, type, 0, rowNum, false); scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); for (int32_t i = 0; i < rowNum; ++i) { - colDataAppend(pInput->columnData, i, (const char *)&val_float[i], false); + colDataSetVal(pInput->columnData, i, (const char *)&val_float[i], false); PRINTF("float before SIN:%f\n", *((float *)colDataGetData(pInput->columnData, i))); } @@ -2556,7 +2556,7 @@ TEST(ScalarFunctionTest, cosFunction_column) { scltMakeDataBlock(&pInput, type, 0, rowNum, false); scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); for (int32_t i = 0; i < rowNum; ++i) { - colDataAppend(pInput->columnData, i, (const char *)&val_tinyint[i], false); + colDataSetVal(pInput->columnData, i, (const char *)&val_tinyint[i], false); PRINTF("tiny_int before COS:%d\n", *(int8_t *)colDataGetData(pInput->columnData, i)); } @@ -2575,7 +2575,7 @@ TEST(ScalarFunctionTest, cosFunction_column) { scltMakeDataBlock(&pInput, type, 0, rowNum, false); scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); for (int32_t i = 0; i < rowNum; ++i) { - colDataAppend(pInput->columnData, i, (const char *)&val_float[i], false); + colDataSetVal(pInput->columnData, i, (const char *)&val_float[i], false); PRINTF("float before COS:%f\n", *(float *)colDataGetData(pInput->columnData, i)); } @@ -2646,7 +2646,7 @@ TEST(ScalarFunctionTest, tanFunction_column) { scltMakeDataBlock(&pInput, type, 0, rowNum, false); scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); for (int32_t i = 0; i < rowNum; ++i) { - colDataAppend(pInput->columnData, i, (const char *)&val_tinyint[i], false); + colDataSetVal(pInput->columnData, i, (const char *)&val_tinyint[i], false); PRINTF("tiny_int before TAN:%d\n", *((int8_t *)colDataGetData(pInput->columnData, i))); } @@ -2665,7 +2665,7 @@ TEST(ScalarFunctionTest, tanFunction_column) { scltMakeDataBlock(&pInput, type, 0, rowNum, false); scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); for (int32_t i = 0; i < rowNum; ++i) { - colDataAppend(pInput->columnData, i, (const char *)&val_float[i], false); + colDataSetVal(pInput->columnData, i, (const char *)&val_float[i], false); PRINTF("float before TAN:%f\n", *((float *)colDataGetData(pInput->columnData, i))); } @@ -2736,7 +2736,7 @@ TEST(ScalarFunctionTest, asinFunction_column) { scltMakeDataBlock(&pInput, type, 0, rowNum, false); scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); for (int32_t i = 0; i < rowNum; ++i) { - colDataAppend(pInput->columnData, i, (const char *)&val_tinyint[i], false); + colDataSetVal(pInput->columnData, i, (const char *)&val_tinyint[i], false); PRINTF("tiny_int before ASIN:%d\n", *((int8_t *)colDataGetData(pInput->columnData, i))); } @@ -3395,7 +3395,7 @@ TEST(ScalarFunctionTest, logFunction_column) { scltMakeDataBlock(&input[i], type, 0, rowNum, false); pInput[i] = *input[i]; for (int32_t j = 0; j < rowNum; ++j) { - colDataAppend(pInput[i].columnData, j, (const char *)&val_tinyint[i][j], false); + colDataSetVal(pInput[i].columnData, j, (const char *)&val_tinyint[i][j], false); } PRINTF("tiny_int before LOG:%d,%d,%d\n", *((int8_t *)pInput[i].data + 0), *((int8_t *)pInput[i].data + 1), *((int8_t *)pInput[i].data + 2)); @@ -3419,7 +3419,7 @@ TEST(ScalarFunctionTest, logFunction_column) { scltMakeDataBlock(&input[i], type, 0, rowNum, false); pInput[i] = *input[i]; for (int32_t j = 0; j < rowNum; ++j) { - colDataAppend(pInput[i].columnData, j, (const char *)&val_float[i][j], false); + colDataSetVal(pInput[i].columnData, j, (const char *)&val_float[i][j], false); } PRINTF("float before LOG:%f,%f,%f\n", *((float *)colDataGetData(pInput[i], 0)), *((float *)colDataGetData(pInput[i], 1)), *((float *)colDataGetData(pInput[i], 2))); @@ -3442,12 +3442,12 @@ TEST(ScalarFunctionTest, logFunction_column) { scltMakeDataBlock(&input[0], TSDB_DATA_TYPE_TINYINT, 0, rowNum, false); pInput[0] = *input[0]; for (int32_t i = 0; i < rowNum; ++i) { - colDataAppend(pInput[0].columnData, i, (const char *)¶m0[i], false); + colDataSetVal(pInput[0].columnData, i, (const char *)¶m0[i], false); } scltMakeDataBlock(&input[1], TSDB_DATA_TYPE_FLOAT, 0, rowNum, false); pInput[1] = *input[1]; for (int32_t i = 0; i < rowNum; ++i) { - colDataAppend(pInput[1].columnData, i, (const char *)¶m1[i], false); + colDataSetVal(pInput[1].columnData, i, (const char *)¶m1[i], false); } PRINTF("tiny_int, float before LOG:{%d,%f}, {%d,%f}, {%d,%f}\n", *((int8_t *)pInput[0].data + 0), *((float *)pInput[1].data + 0), *((int8_t *)pInput[0].data + 1), *((float *)pInput[1].data + 1), @@ -3558,7 +3558,7 @@ TEST(ScalarFunctionTest, powFunction_column) { scltMakeDataBlock(&input[i], type, 0, rowNum, false); pInput[i] = *input[i]; for (int32_t j = 0; j < rowNum; ++j) { - colDataAppend(pInput[i].columnData, j, (const char *)&val_tinyint[i][j], false); + colDataSetVal(pInput[i].columnData, j, (const char *)&val_tinyint[i][j], false); } PRINTF("tiny_int before POW:%d,%d,%d\n", *((int8_t *)pInput[i].data + 0), *((int8_t *)pInput[i].data + 1), *((int8_t *)pInput[i].data + 2)); @@ -3583,7 +3583,7 @@ TEST(ScalarFunctionTest, powFunction_column) { scltMakeDataBlock(&input[i], type, 0, rowNum, false); pInput[i] = *input[i]; for (int32_t j = 0; j < rowNum; ++j) { - colDataAppend(pInput[i].columnData, j, (const char *)&val_float[i][j], false); + colDataSetVal(pInput[i].columnData, j, (const char *)&val_float[i][j], false); } PRINTF("float before POW:%f,%f,%f\n", *((float *)pInput[i].data + 0), *((float *)pInput[i].data + 1), *((float *)pInput[i].data + 2)); @@ -3606,12 +3606,12 @@ TEST(ScalarFunctionTest, powFunction_column) { scltMakeDataBlock(&input[0], TSDB_DATA_TYPE_TINYINT, 0, rowNum, false); pInput[0] = *input[0]; for (int32_t i = 0; i < rowNum; ++i) { - colDataAppend(pInput[0].columnData, i, (const char *)¶m0[i], false); + colDataSetVal(pInput[0].columnData, i, (const char *)¶m0[i], false); } scltMakeDataBlock(&input[1], TSDB_DATA_TYPE_FLOAT, 0, rowNum, false); pInput[1] = *input[1]; for (int32_t i = 0; i < rowNum; ++i) { - colDataAppend(pInput[1].columnData, i, (const char *)¶m1[i], false); + colDataSetVal(pInput[1].columnData, i, (const char *)¶m1[i], false); } PRINTF("tiny_int, float before POW:{%d,%f}, {%d,%f}, {%d,%f}\n", *((int8_t *)pInput[0].data + 0), *((float *)pInput[1].data + 0), *((int8_t *)pInput[0].data + 1), *((float *)pInput[1].data + 1), diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index ffb600be6e..8baebaee42 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -17,7 +17,7 @@ int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock* pData) { int32_t blockNum = pReq->blockNum; - SArray* pArray = taosArrayInit_s(blockNum, sizeof(SSDataBlock), blockNum); + SArray* pArray = taosArrayInit_s(sizeof(SSDataBlock), blockNum); if (pArray == NULL) { return -1; } diff --git a/source/libs/sync/inc/syncPipeline.h b/source/libs/sync/inc/syncPipeline.h index 504a9f0bd7..a823cfda0b 100644 --- a/source/libs/sync/inc/syncPipeline.h +++ b/source/libs/sync/inc/syncPipeline.h @@ -99,6 +99,7 @@ int32_t syncLogBufferReInit(SSyncLogBuffer* pBuf, SSyncNode* pNode); // access int64_t syncLogBufferGetEndIndex(SSyncLogBuffer* pBuf); SyncTerm syncLogBufferGetLastMatchTerm(SSyncLogBuffer* pBuf); +bool syncLogBufferIsEmpty(SSyncLogBuffer* pBuf); int32_t syncLogBufferAppend(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEntry* pEntry); int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEntry* pEntry, SyncTerm prevTerm); int64_t syncLogBufferProceed(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncTerm* pMatchTerm); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 5935f8393a..a9571f9e3b 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2479,6 +2479,10 @@ int32_t syncNodeOnLocalCmd(SSyncNode* ths, const SRpcMsg* pRpcMsg) { syncNodeStepDown(ths, pMsg->currentTerm); } else if (pMsg->cmd == SYNC_LOCAL_CMD_FOLLOWER_CMT) { + if (syncLogBufferIsEmpty(ths->pLogBuf)) { + sError("vgId:%d, sync log buffer is empty.", ths->vgId); + return 0; + } SyncTerm matchTerm = syncLogBufferGetLastMatchTerm(ths->pLogBuf); if (pMsg->currentTerm == matchTerm) { (void)syncNodeUpdateCommitIndex(ths, pMsg->commitIndex); diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index c9ff2d2dcc..e2b039a2e4 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -253,6 +253,7 @@ int32_t syncLogBufferInit(SSyncLogBuffer* pBuf, SSyncNode* pNode) { int32_t syncLogBufferReInit(SSyncLogBuffer* pBuf, SSyncNode* pNode) { taosThreadMutexLock(&pBuf->mutex); + syncLogBufferValidate(pBuf); for (SyncIndex index = pBuf->startIndex; index < pBuf->endIndex; index++) { SSyncRaftEntry* pEntry = pBuf->entries[(index + pBuf->size) % pBuf->size].pItem; if (pEntry == NULL) continue; @@ -265,6 +266,7 @@ int32_t syncLogBufferReInit(SSyncLogBuffer* pBuf, SSyncNode* pNode) { if (ret < 0) { sError("vgId:%d, failed to re-initialize sync log buffer since %s.", pNode->vgId, terrstr()); } + syncLogBufferValidate(pBuf); taosThreadMutexUnlock(&pBuf->mutex); return ret; } @@ -283,6 +285,13 @@ SyncTerm syncLogBufferGetLastMatchTerm(SSyncLogBuffer* pBuf) { return term; } +bool syncLogBufferIsEmpty(SSyncLogBuffer* pBuf) { + taosThreadMutexLock(&pBuf->mutex); + bool empty = (pBuf->endIndex <= pBuf->startIndex); + taosThreadMutexUnlock(&pBuf->mutex); + return empty; +} + int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEntry* pEntry, SyncTerm prevTerm) { taosThreadMutexLock(&pBuf->mutex); syncLogBufferValidate(pBuf); @@ -1073,6 +1082,7 @@ int32_t syncLogBufferRollback(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncIndex int32_t syncLogBufferReset(SSyncLogBuffer* pBuf, SSyncNode* pNode) { taosThreadMutexLock(&pBuf->mutex); + syncLogBufferValidate(pBuf); SyncIndex lastVer = pNode->pLogStore->syncLogLastIndex(pNode->pLogStore); ASSERT(lastVer == pBuf->matchIndex); SyncIndex index = pBuf->endIndex - 1; @@ -1089,6 +1099,7 @@ int32_t syncLogBufferReset(SSyncLogBuffer* pBuf, SSyncNode* pNode) { SSyncLogReplMgr* pMgr = pNode->logReplMgrs[i]; syncLogReplMgrReset(pMgr); } + syncLogBufferValidate(pBuf); taosThreadMutexUnlock(&pBuf->mutex); return 0; } diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 4cd6a39bfe..4f6e3cf8ed 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -2270,6 +2270,9 @@ int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { } else { lidx = lidx + 1; } + if (TDB_CELLDECODER_FREE_KEY(&pBtc->coder)) { + tdbFree((void*)pTKey); + } // compare last cell if (lidx <= ridx) { @@ -2281,6 +2284,9 @@ int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { } else { ridx = ridx - 1; } + if (TDB_CELLDECODER_FREE_KEY(&pBtc->coder)) { + tdbFree((void*)pTKey); + } } // binary search @@ -2291,6 +2297,9 @@ int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { pBtc->idx = (lidx + ridx) >> 1; tdbBtcGet(pBtc, &pTKey, &tkLen, NULL, NULL); c = pBt->kcmpr(pKey, kLen, pTKey, tkLen); + if (TDB_CELLDECODER_FREE_KEY(&pBtc->coder)) { + tdbFree((void*)pTKey); + } if (c < 0) { // pKey < cd.pKey ridx = pBtc->idx - 1; diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 16ea25a41a..f5f3b52f50 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -13,8 +13,6 @@ * along with this program. If not, see . */ -#ifdef USE_UV - #include "transComm.h" void* (*taosInitHandle[])(uint32_t ip, uint32_t port, char* label, int32_t numOfThreads, void* fp, void* shandle) = { @@ -88,6 +86,11 @@ void* rpcOpen(const SRpcInit* pInit) { pRpc->connType = pInit->connType; pRpc->idleTime = pInit->idleTime; + pRpc->parent = pInit->parent; + if (pInit->user) { + tstrncpy(pRpc->user, pInit->user, sizeof(pRpc->user)); + } + pRpc->tcphandle = (*taosInitHandle[pRpc->connType])(ip, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); @@ -95,10 +98,6 @@ void* rpcOpen(const SRpcInit* pInit) { taosMemoryFree(pRpc); return NULL; } - pRpc->parent = pInit->parent; - if (pInit->user) { - tstrncpy(pRpc->user, pInit->user, sizeof(pRpc->user)); - } int64_t refId = transAddExHandle(transGetInstMgt(), pRpc); transAcquireExHandle(transGetInstMgt(), refId); @@ -188,5 +187,3 @@ void rpcCleanup(void) { return; } - -#endif diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 7e1aeafaad..5d6751a260 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -727,7 +727,7 @@ static SCliConn* cliCreateConn(SCliThrd* pThrd) { QUEUE_INIT(&conn->q); conn->hostThrd = pThrd; conn->status = ConnNormal; - conn->broken = 0; + conn->broken = false; transRefCliHandle(conn); atomic_add_fetch_32(&pThrd->connCount, 1); @@ -985,6 +985,7 @@ _RETURN: } static void cliDestroyBatch(SCliBatch* pBatch) { + if (pBatch == NULL) return; while (!QUEUE_IS_EMPTY(&pBatch->wq)) { queue* h = QUEUE_HEAD(&pBatch->wq); QUEUE_REMOVE(h); @@ -997,6 +998,11 @@ static void cliDestroyBatch(SCliBatch* pBatch) { taosMemoryFree(pBatch); } static void cliHandleBatchReq(SCliBatch* pBatch, SCliThrd* pThrd) { + if (pThrd->quit == true) { + cliDestroyBatch(pBatch); + return; + } + if (pBatch == NULL || pBatch->wLen == 0 || QUEUE_IS_EMPTY(&pBatch->wq)) { return; } @@ -1082,17 +1088,23 @@ static void cliSendBatchCb(uv_write_t* req, int status) { if (status != 0) { tDebug("%s conn %p failed to send batch msg, batch size:%d, msgLen:%d, reason:%s", CONN_GET_INST_LABEL(conn), conn, p->wLen, p->batchSize, uv_err_name(status)); - cliHandleExcept(conn); + + if (!uv_is_closing((uv_handle_t*)&conn->stream)) cliHandleExcept(conn); + cliHandleBatchReq(nxtBatch, thrd); } else { tDebug("%s conn %p succ to send batch msg, batch size:%d, msgLen:%d", CONN_GET_INST_LABEL(conn), conn, p->wLen, p->batchSize); - - if (nxtBatch != NULL) { - conn->pBatch = nxtBatch; - cliSendBatch(conn); + if (!uv_is_closing((uv_handle_t*)&conn->stream)) { + if (nxtBatch != NULL) { + conn->pBatch = nxtBatch; + cliSendBatch(conn); + } else { + addConnToPool(thrd->pool, conn); + } } else { - addConnToPool(thrd->pool, conn); + cliDestroyBatch(nxtBatch); + // conn release by other callback } } @@ -1454,6 +1466,11 @@ static void cliNoBatchDealReq(queue* wq, SCliThrd* pThrd) { QUEUE_REMOVE(h); SCliMsg* pMsg = QUEUE_DATA(h, SCliMsg, q); + + if (pMsg->type == Quit) { + pThrd->stopMsg = pMsg; + continue; + } (*cliAsyncHandle[pMsg->type])(pMsg, pThrd); count++; @@ -1485,6 +1502,12 @@ static void cliBatchDealReq(queue* wq, SCliThrd* pThrd) { QUEUE_REMOVE(h); SCliMsg* pMsg = QUEUE_DATA(h, SCliMsg, q); + + if (pMsg->type == Quit) { + pThrd->stopMsg = pMsg; + continue; + } + if (pMsg->type == Normal && REQUEST_NO_RESP(&pMsg->msg)) { STransConnCtx* pCtx = pMsg->ctx; @@ -1582,7 +1605,6 @@ static void cliAsyncCb(uv_async_t* handle) { SCliThrd* pThrd = item->pThrd; STrans* pTransInst = pThrd->pTransInst; - SCliMsg* pMsg = NULL; // batch process to avoid to lock/unlock frequently queue wq; taosThreadMutexLock(&item->mtx); @@ -2285,24 +2307,11 @@ int transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STran transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); return TSDB_CODE_RPC_BROKEN_LINK; } - /*if (pTransInst->connLimitNum > 0 && REQUEST_NO_RESP(pReq)) { - char key[TSDB_FQDN_LEN + 64] = {0}; - char* ip = EPSET_GET_INUSE_IP((SEpSet*)pEpSet); - uint16_t port = EPSET_GET_INUSE_PORT((SEpSet*)pEpSet); - CONN_CONSTRUCT_HASH_KEY(key, ip, port); - - int32_t* val = taosHashGet(pThrd->connLimitCache, key, strlen(key)); - if (val != NULL && *val >= pTransInst->connLimitNum) { - transFreeMsg(pReq->pCont); - transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); - return TSDB_CODE_RPC_MAX_SESSIONS; - } - }*/ TRACE_SET_MSGID(&pReq->info.traceId, tGenIdPI64()); - STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx)); pCtx->epSet = *pEpSet; + pCtx->origEpSet = *pEpSet; pCtx->ahandle = pReq->info.ahandle; pCtx->msgType = pReq->msgType; diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 4c107a88f1..0dfc7677b3 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -12,7 +12,6 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#ifdef USE_UV #include "transComm.h" @@ -67,7 +66,7 @@ int32_t transDecompressMsg(char** msg, int32_t len) { char* buf = taosMemoryCalloc(1, oriLen + sizeof(STransMsgHead)); STransMsgHead* pNewHead = (STransMsgHead*)buf; - int32_t decompLen = LZ4_decompress_safe(pCont + sizeof(STransCompMsg), pNewHead->content, + int32_t decompLen = LZ4_decompress_safe(pCont + sizeof(STransCompMsg), (char*)pNewHead->content, len - sizeof(STransMsgHead) - sizeof(STransCompMsg), oriLen); memcpy((char*)pNewHead, (char*)pHead, sizeof(STransMsgHead)); @@ -655,4 +654,3 @@ void transDestoryExHandle(void* handle) { } taosMemoryFree(handle); } -#endif diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 0bae81f119..264bf53800 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -48,7 +48,7 @@ SArray* taosArrayInit(size_t size, size_t elemSize) { return pArray; } -SArray* taosArrayInit_s(size_t size, size_t elemSize, size_t initialSize) { +SArray* taosArrayInit_s(size_t elemSize, size_t initialSize) { SArray* pArray = taosMemoryMalloc(sizeof(SArray)); if (pArray == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 695a83abb1..7631da1b56 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -327,23 +327,26 @@ int32_t tsDecompressINTImp(const char *const input, const int32_t nelements, cha // calculate the cumulative sum (prefix sum) for each number // decode[0] = prev_value + final[0] // decode[1] = decode[0] + final[1] -----> prev_value + final[0] + final[1] - // decode[2] = decode[1] + final[1] -----> prev_value + final[0] + final[1] + final[2] - // decode[3] = decode[2] + final[1] -----> prev_value + final[0] + final[1] + final[2] + final[3] + // decode[2] = decode[1] + final[2] -----> prev_value + final[0] + final[1] + final[2] + // decode[3] = decode[2] + final[3] -----> prev_value + final[0] + final[1] + final[2] + final[3] // 1, 2, 3, 4 - //+ 0, 1, 2, 3 - // 1, 3, 5, 7 + //+ 0, 1, 0, 3 + // 1, 3, 3, 7 // shift and add for the first round __m128i prev = _mm_set1_epi64x(prev_value); - delta = _mm256_add_epi64(delta, _mm256_slli_si256(delta, 8)); + __m256i x = _mm256_slli_si256(delta, 8); + + delta = _mm256_add_epi64(delta, x); _mm256_storeu_si256((__m256i *)&p[_pos], delta); - // 1, 3, 5, 7 - //+ 0, 0, 1, 3 + // 1, 3, 3, 7 + //+ 0, 0, 3, 3 // 1, 3, 6, 10 // shift and add operation for the second round __m128i firstPart = _mm_loadu_si128((__m128i *)&p[_pos]); - __m128i secPart = _mm_add_epi64(_mm_loadu_si128((__m128i *)&p[_pos + 2]), firstPart); + __m128i secondItem = _mm_set1_epi64x(p[_pos + 1]); + __m128i secPart = _mm_add_epi64(_mm_loadu_si128((__m128i *)&p[_pos + 2]), secondItem); firstPart = _mm_add_epi64(firstPart, prev); secPart = _mm_add_epi64(secPart, prev); @@ -353,15 +356,18 @@ int32_t tsDecompressINTImp(const char *const input, const int32_t nelements, cha shiftBits = _mm256_add_epi64(shiftBits, inc); prev_value = p[_pos + 3]; +// uDebug("_pos:%d %"PRId64", %"PRId64", %"PRId64", %"PRId64, _pos, p[_pos], p[_pos+1], p[_pos+2], p[_pos+3]); _pos += 4; } // handle the remain value for (int32_t i = 0; i < remain; i++) { - zigzag_value = ((w >> (v + (batch * bit))) & mask); + zigzag_value = ((w >> (v + (batch * bit * 4))) & mask); prev_value += ZIGZAG_DECODE(int64_t, zigzag_value); p[_pos++] = prev_value; +// uDebug("_pos:%d %"PRId64, _pos-1, p[_pos-1]); + v += bit; } } else { @@ -370,6 +376,8 @@ int32_t tsDecompressINTImp(const char *const input, const int32_t nelements, cha prev_value += ZIGZAG_DECODE(int64_t, zigzag_value); p[_pos++] = prev_value; +// uDebug("_pos:%d %"PRId64, _pos-1, p[_pos-1]); + v += bit; } } diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index 27d14d05b1..33182eca49 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -325,7 +325,7 @@ int32_t tjsonToTArray(const SJson* pJson, const char* pName, FToObject func, SAr const cJSON* jArray = tjsonGetObjectItem(pJson, pName); int32_t size = tjsonGetArraySize(jArray); if (size > 0) { - *pArray = taosArrayInit_s(size, itemSize, size); + *pArray = taosArrayInit_s(itemSize, size); if (NULL == *pArray) { return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/util/src/tutil.c b/source/util/src/tutil.c index 55d7d4f6e7..1f9ca7407e 100644 --- a/source/util/src/tutil.c +++ b/source/util/src/tutil.c @@ -118,7 +118,7 @@ char **strsplit(char *z, const char *delim, int32_t *num) { if ((*num) >= size) { size = (size << 1); split = taosMemoryRealloc(split, POINTER_BYTES * size); - ASSERTS(NULL != split, "realloc memory failed. size=%d", POINTER_BYTES * size); + ASSERTS(NULL != split, "realloc memory failed. size=%d", (int32_t) POINTER_BYTES * size); } } diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index 5581931178..631bcb443e 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -22,7 +22,7 @@ typedef void *(*ThreadFp)(void *param); int32_t tQWorkerInit(SQWorkerPool *pool) { pool->qset = taosOpenQset(); - pool->workers = taosMemoryCalloc(pool->max, sizeof(SQWorker)); + pool->workers = taosMemoryCalloc(pool->max, sizeof(SQueueWorker)); if (pool->workers == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -31,7 +31,7 @@ int32_t tQWorkerInit(SQWorkerPool *pool) { (void)taosThreadMutexInit(&pool->mutex, NULL); for (int32_t i = 0; i < pool->max; ++i) { - SQWorker *worker = pool->workers + i; + SQueueWorker *worker = pool->workers + i; worker->id = i; worker->pool = pool; } @@ -42,14 +42,14 @@ int32_t tQWorkerInit(SQWorkerPool *pool) { void tQWorkerCleanup(SQWorkerPool *pool) { for (int32_t i = 0; i < pool->max; ++i) { - SQWorker *worker = pool->workers + i; + SQueueWorker *worker = pool->workers + i; if (taosCheckPthreadValid(worker->thread)) { taosQsetThreadResume(pool->qset); } } for (int32_t i = 0; i < pool->max; ++i) { - SQWorker *worker = pool->workers + i; + SQueueWorker *worker = pool->workers + i; if (taosCheckPthreadValid(worker->thread)) { uInfo("worker:%s:%d is stopping", pool->name, worker->id); taosThreadJoin(worker->thread, NULL); @@ -65,7 +65,7 @@ void tQWorkerCleanup(SQWorkerPool *pool) { uInfo("worker:%s is closed", pool->name); } -static void *tQWorkerThreadFp(SQWorker *worker) { +static void *tQWorkerThreadFp(SQueueWorker *worker) { SQWorkerPool *pool = worker->pool; SQueueInfo qinfo = {0}; void *msg = NULL; @@ -106,7 +106,7 @@ STaosQueue *tQWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp) { // spawn a thread to process queue if (pool->num < pool->max) { do { - SQWorker *worker = pool->workers + pool->num; + SQueueWorker *worker = pool->workers + pool->num; TdThreadAttr thAttr; taosThreadAttrInit(&thAttr); @@ -138,7 +138,7 @@ void tQWorkerFreeQueue(SQWorkerPool *pool, STaosQueue *queue) { int32_t tAutoQWorkerInit(SAutoQWorkerPool *pool) { pool->qset = taosOpenQset(); - pool->workers = taosArrayInit(2, sizeof(SQWorker *)); + pool->workers = taosArrayInit(2, sizeof(SQueueWorker *)); if (pool->workers == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -153,14 +153,14 @@ int32_t tAutoQWorkerInit(SAutoQWorkerPool *pool) { void tAutoQWorkerCleanup(SAutoQWorkerPool *pool) { int32_t size = taosArrayGetSize(pool->workers); for (int32_t i = 0; i < size; ++i) { - SQWorker *worker = taosArrayGetP(pool->workers, i); + SQueueWorker *worker = taosArrayGetP(pool->workers, i); if (taosCheckPthreadValid(worker->thread)) { taosQsetThreadResume(pool->qset); } } for (int32_t i = 0; i < size; ++i) { - SQWorker *worker = taosArrayGetP(pool->workers, i); + SQueueWorker *worker = taosArrayGetP(pool->workers, i); if (taosCheckPthreadValid(worker->thread)) { uInfo("worker:%s:%d is stopping", pool->name, worker->id); taosThreadJoin(worker->thread, NULL); @@ -177,7 +177,7 @@ void tAutoQWorkerCleanup(SAutoQWorkerPool *pool) { uInfo("worker:%s is closed", pool->name); } -static void *tAutoQWorkerThreadFp(SQWorker *worker) { +static void *tAutoQWorkerThreadFp(SQueueWorker *worker) { SAutoQWorkerPool *pool = worker->pool; SQueueInfo qinfo = {0}; void *msg = NULL; @@ -222,7 +222,7 @@ STaosQueue *tAutoQWorkerAllocQueue(SAutoQWorkerPool *pool, void *ahandle, FItem // spawn a thread to process queue while (curWorkerNum < dstWorkerNum) { - SQWorker *worker = taosMemoryCalloc(1, sizeof(SQWorker)); + SQueueWorker *worker = taosMemoryCalloc(1, sizeof(SQueueWorker)); if (worker == NULL || taosArrayPush(pool->workers, &worker) == NULL) { uError("worker:%s:%d failed to create", pool->name, curWorkerNum); taosMemoryFree(worker); diff --git a/source/util/src/xxhash.c b/source/util/src/xxhash.c deleted file mode 100644 index ff28749e31..0000000000 --- a/source/util/src/xxhash.c +++ /dev/null @@ -1,1030 +0,0 @@ -/* -* xxHash - Fast Hash algorithm -* Copyright (C) 2012-2016, Yann Collet -* -* BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) -* -* Redistribution and use in source and binary forms, with or without -* modification, are permitted provided that the following conditions are -* met: -* -* * Redistributions of source code must retain the above copyright -* notice, this list of conditions and the following disclaimer. -* * Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following disclaimer -* in the documentation and/or other materials provided with the -* distribution. -* -* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -* -* You can contact the author at : -* - xxHash homepage: http://www.xxhash.com -* - xxHash source repository : https://github.com/Cyan4973/xxHash -*/ - - -/* ************************************* -* Tuning parameters -***************************************/ -/*!XXH_FORCE_MEMORY_ACCESS : - * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. - * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. - * The below switch allow to select different access method for improved performance. - * Method 0 (default) : use `memcpy()`. Safe and portable. - * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). - * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. - * Method 2 : direct access. This method doesn't depend on compiler but violate C standard. - * It can generate buggy code on targets which do not support unaligned memory accesses. - * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) - * See http://stackoverflow.com/a/32095106/646947 for details. - * Prefer these methods in priority order (0 > 1 > 2) - */ -#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ -# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) \ - || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) \ - || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) -# define XXH_FORCE_MEMORY_ACCESS 2 -# elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || \ - (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) \ - || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) \ - || defined(__ARM_ARCH_7S__) )) -# define XXH_FORCE_MEMORY_ACCESS 1 -# endif -#endif - -/*!XXH_ACCEPT_NULL_INPUT_POINTER : - * If input pointer is NULL, xxHash default behavior is to dereference it, triggering a segfault. - * When this macro is enabled, xxHash actively checks input for null pointer. - * It it is, result for null input pointers is the same as a null-length input. - */ -#ifndef XXH_ACCEPT_NULL_INPUT_POINTER /* can be defined externally */ -# define XXH_ACCEPT_NULL_INPUT_POINTER 0 -#endif - -/*!XXH_FORCE_NATIVE_FORMAT : - * By default, xxHash library provides endian-independent Hash values, based on little-endian convention. - * Results are therefore identical for little-endian and big-endian CPU. - * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. - * Should endian-independence be of no importance for your application, you may set the #define below to 1, - * to improve speed for Big-endian CPU. - * This option has no impact on Little_Endian CPU. - */ -#ifndef XXH_FORCE_NATIVE_FORMAT /* can be defined externally */ -# define XXH_FORCE_NATIVE_FORMAT 0 -#endif - -/*!XXH_FORCE_ALIGN_CHECK : - * This is a minor performance trick, only useful with lots of very small keys. - * It means : check for aligned/unaligned input. - * The check costs one initial branch per hash; - * set it to 0 when the input is guaranteed to be aligned, - * or when alignment doesn't matter for performance. - */ -#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ -# if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) -# define XXH_FORCE_ALIGN_CHECK 0 -# else -# define XXH_FORCE_ALIGN_CHECK 1 -# endif -#endif - - -/* ************************************* -* Includes & Memory related functions -***************************************/ -/*! Modify the local functions below should you wish to use some other memory routines -* for malloc(), free() */ -#include -static void* XXH_malloc(size_t s) { return malloc(s); } -static void XXH_free (void* p) { free(p); } -/*! and for memcpy() */ -#include -static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } - -#include /* assert */ - -#define XXH_STATIC_LINKING_ONLY -#include "xxhash.h" - - -/* ************************************* -* Compiler Specific Options -***************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# define FORCE_INLINE static __forceinline -#else -# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif -# else -# define FORCE_INLINE static -# endif /* __STDC_VERSION__ */ -#endif - - -/* ************************************* -* Basic Types -***************************************/ -#ifndef MEM_MODULE -# if !defined (__VMS) \ - && (defined (__cplusplus) \ - || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) -# include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; -# else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; -# endif -#endif - -#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) - -/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ -static U32 XXH_read32(const void* memPtr) { return *(const U32*) memPtr; } - -#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) - -/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ -/* currently only defined for gcc and icc */ -typedef union { U32 u32; } __attribute__((packed)) unalign; -static U32 XXH_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } - -#else - -/* portable and safe solution. Generally efficient. - * see : http://stackoverflow.com/a/32095106/646947 - */ -static U32 XXH_read32(const void* memPtr) -{ - U32 val; - memcpy(&val, memPtr, sizeof(val)); - return val; -} - -#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ - - -/* **************************************** -* Compiler-specific Functions and Macros -******************************************/ -#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) - -/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */ -#if defined(_MSC_VER) -# define XXH_rotl32(x,r) _rotl(x,r) -# define XXH_rotl64(x,r) _rotl64(x,r) -#else -# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) -# define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r))) -#endif - -#if defined(_MSC_VER) /* Visual Studio */ -# define XXH_swap32 _byteswap_ulong -#elif XXH_GCC_VERSION >= 403 -# define XXH_swap32 __builtin_bswap32 -#else -static U32 XXH_swap32 (U32 x) -{ - return ((x << 24) & 0xff000000 ) | - ((x << 8) & 0x00ff0000 ) | - ((x >> 8) & 0x0000ff00 ) | - ((x >> 24) & 0x000000ff ); -} -#endif - - -/* ************************************* -* Architecture Macros -***************************************/ -typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; - -/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */ -#ifndef XXH_CPU_LITTLE_ENDIAN -static int XXH_isLittleEndian(void) -{ - const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ - return one.c[0]; -} -# define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian() -#endif - - -/* *************************** -* Memory reads -*****************************/ -typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; - -FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) -{ - if (align==XXH_unaligned) - return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); - else - return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr); -} - -FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) -{ - return XXH_readLE32_align(ptr, endian, XXH_unaligned); -} - -static U32 XXH_readBE32(const void* ptr) -{ - return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); -} - - -/* ************************************* -* Macros -***************************************/ -#define XXH_STATIC_ASSERT(c) { enum { XXH_sa = 1/(int)(!!(c)) }; } /* use after variable declarations */ -XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } - - -/* ******************************************************************* -* 32-bit hash functions -*********************************************************************/ -static const U32 PRIME32_1 = 2654435761U; -static const U32 PRIME32_2 = 2246822519U; -static const U32 PRIME32_3 = 3266489917U; -static const U32 PRIME32_4 = 668265263U; -static const U32 PRIME32_5 = 374761393U; - -static U32 XXH32_round(U32 seed, U32 input) -{ - seed += input * PRIME32_2; - seed = XXH_rotl32(seed, 13); - seed *= PRIME32_1; - return seed; -} - -/* mix all bits */ -static U32 XXH32_avalanche(U32 h32) -{ - h32 ^= h32 >> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; - return(h32); -} - -#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align) - -static U32 -XXH32_finalize(U32 h32, const void* ptr, size_t len, - XXH_endianess endian, XXH_alignment align) - -{ - const BYTE* p = (const BYTE*)ptr; - -#define PROCESS1 \ - h32 += (*p++) * PRIME32_5; \ - h32 = XXH_rotl32(h32, 11) * PRIME32_1 ; - -#define PROCESS4 \ - h32 += XXH_get32bits(p) * PRIME32_3; \ - p+=4; \ - h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; - - switch(len&15) /* or switch(bEnd - p) */ - { - case 12: PROCESS4; - /* fallthrough */ - case 8: PROCESS4; - /* fallthrough */ - case 4: PROCESS4; - return XXH32_avalanche(h32); - - case 13: PROCESS4; - /* fallthrough */ - case 9: PROCESS4; - /* fallthrough */ - case 5: PROCESS4; - PROCESS1; - return XXH32_avalanche(h32); - - case 14: PROCESS4; - /* fallthrough */ - case 10: PROCESS4; - /* fallthrough */ - case 6: PROCESS4; - PROCESS1; - PROCESS1; - return XXH32_avalanche(h32); - - case 15: PROCESS4; - /* fallthrough */ - case 11: PROCESS4; - /* fallthrough */ - case 7: PROCESS4; - /* fallthrough */ - case 3: PROCESS1; - /* fallthrough */ - case 2: PROCESS1; - /* fallthrough */ - case 1: PROCESS1; - /* fallthrough */ - case 0: return XXH32_avalanche(h32); - } - assert(0); - return h32; /* reaching this point is deemed impossible */ -} - - -FORCE_INLINE U32 -XXH32_endian_align(const void* input, size_t len, U32 seed, - XXH_endianess endian, XXH_alignment align) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* bEnd = p + len; - U32 h32; - -#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) - if (p==NULL) { - len=0; - bEnd=p=(const BYTE*)(size_t)16; - } -#endif - - if (len>=16) { - const BYTE* const limit = bEnd - 15; - U32 v1 = seed + PRIME32_1 + PRIME32_2; - U32 v2 = seed + PRIME32_2; - U32 v3 = seed + 0; - U32 v4 = seed - PRIME32_1; - - do { - v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4; - v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4; - v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4; - v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4; - } while (p < limit); - - h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) - + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); - } else { - h32 = seed + PRIME32_5; - } - - h32 += (U32)len; - - return XXH32_finalize(h32, p, len&15, endian, align); -} - - -XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed) -{ -#if 0 - /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ - XXH32_state_t state; - XXH32_reset(&state, seed); - XXH32_update(&state, input, len); - return XXH32_digest(&state); -#else - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if (XXH_FORCE_ALIGN_CHECK) { - if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); - else - return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); - } } - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); - else - return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); -#endif -} - - - -/*====== Hash streaming ======*/ - -XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) -{ - return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); -} -XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) -{ - XXH_free(statePtr); - return XXH_OK; -} - -XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState) -{ - memcpy(dstState, srcState, sizeof(*dstState)); -} - -XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed) -{ - XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ - memset(&state, 0, sizeof(state)); - state.v1 = seed + PRIME32_1 + PRIME32_2; - state.v2 = seed + PRIME32_2; - state.v3 = seed + 0; - state.v4 = seed - PRIME32_1; - /* do not write into reserved, planned to be removed in a future version */ - memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved)); - return XXH_OK; -} - - -FORCE_INLINE XXH_errorcode -XXH32_update_endian(XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian) -{ - if (input==NULL) -#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) - return XXH_OK; -#else - return XXH_ERROR; -#endif - - { const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - - state->total_len_32 += (unsigned)len; - state->large_len |= (len>=16) | (state->total_len_32>=16); - - if (state->memsize + len < 16) { /* fill in tmp buffer */ - XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); - state->memsize += (unsigned)len; - return XXH_OK; - } - - if (state->memsize) { /* some data left from previous update */ - XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize); - { const U32* p32 = state->mem32; - state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++; - state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++; - state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++; - state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); - } - p += 16-state->memsize; - state->memsize = 0; - } - - if (p <= bEnd-16) { - const BYTE* const limit = bEnd - 16; - U32 v1 = state->v1; - U32 v2 = state->v2; - U32 v3 = state->v3; - U32 v4 = state->v4; - - do { - v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4; - v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4; - v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4; - v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4; - } while (p<=limit); - - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; - } - - if (p < bEnd) { - XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); - state->memsize = (unsigned)(bEnd-p); - } - } - - return XXH_OK; -} - - -XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_update_endian(state_in, input, len, XXH_littleEndian); - else - return XXH32_update_endian(state_in, input, len, XXH_bigEndian); -} - - -FORCE_INLINE U32 -XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian) -{ - U32 h32; - - if (state->large_len) { - h32 = XXH_rotl32(state->v1, 1) - + XXH_rotl32(state->v2, 7) - + XXH_rotl32(state->v3, 12) - + XXH_rotl32(state->v4, 18); - } else { - h32 = state->v3 /* == seed */ + PRIME32_5; - } - - h32 += state->total_len_32; - - return XXH32_finalize(h32, state->mem32, state->memsize, endian, XXH_aligned); -} - - -XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_digest_endian(state_in, XXH_littleEndian); - else - return XXH32_digest_endian(state_in, XXH_bigEndian); -} - - -/*====== Canonical representation ======*/ - -/*! Default XXH result types are basic unsigned 32 and 64 bits. -* The canonical representation follows human-readable write convention, aka big-endian (large digits first). -* These functions allow transformation of hash result into and from its canonical format. -* This way, hash values can be written into a file or buffer, remaining comparable across different systems. -*/ - -XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) -{ - XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); - if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); - memcpy(dst, &hash, sizeof(*dst)); -} - -XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) -{ - return XXH_readBE32(src); -} - - -#ifndef XXH_NO_LONG_LONG - -/* ******************************************************************* -* 64-bit hash functions -*********************************************************************/ - -/*====== Memory access ======*/ - -#ifndef MEM_MODULE -# define MEM_MODULE -# if !defined (__VMS) \ - && (defined (__cplusplus) \ - || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) -# include - typedef uint64_t U64; -# else - /* if compiler doesn't support unsigned long long, replace by another 64-bit type */ - typedef unsigned long long U64; -# endif -#endif - - -#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) - -/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ -static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; } - -#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) - -/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ -/* currently only defined for gcc and icc */ -typedef union { U32 u32; U64 u64; } __attribute__((packed)) unalign64; -static U64 XXH_read64(const void* ptr) { return ((const unalign64*)ptr)->u64; } - -#else - -/* portable and safe solution. Generally efficient. - * see : http://stackoverflow.com/a/32095106/646947 - */ - -static U64 XXH_read64(const void* memPtr) -{ - U64 val; - memcpy(&val, memPtr, sizeof(val)); - return val; -} - -#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ - -#if defined(_MSC_VER) /* Visual Studio */ -# define XXH_swap64 _byteswap_uint64 -#elif XXH_GCC_VERSION >= 403 -# define XXH_swap64 __builtin_bswap64 -#else -static U64 XXH_swap64 (U64 x) -{ - return ((x << 56) & 0xff00000000000000ULL) | - ((x << 40) & 0x00ff000000000000ULL) | - ((x << 24) & 0x0000ff0000000000ULL) | - ((x << 8) & 0x000000ff00000000ULL) | - ((x >> 8) & 0x00000000ff000000ULL) | - ((x >> 24) & 0x0000000000ff0000ULL) | - ((x >> 40) & 0x000000000000ff00ULL) | - ((x >> 56) & 0x00000000000000ffULL); -} -#endif - -FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) -{ - if (align==XXH_unaligned) - return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); - else - return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr); -} - -FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) -{ - return XXH_readLE64_align(ptr, endian, XXH_unaligned); -} - -static U64 XXH_readBE64(const void* ptr) -{ - return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); -} - - -/*====== xxh64 ======*/ - -static const U64 PRIME64_1 = 11400714785074694791ULL; -static const U64 PRIME64_2 = 14029467366897019727ULL; -static const U64 PRIME64_3 = 1609587929392839161ULL; -static const U64 PRIME64_4 = 9650029242287828579ULL; -static const U64 PRIME64_5 = 2870177450012600261ULL; - -static U64 XXH64_round(U64 acc, U64 input) -{ - acc += input * PRIME64_2; - acc = XXH_rotl64(acc, 31); - acc *= PRIME64_1; - return acc; -} - -static U64 XXH64_mergeRound(U64 acc, U64 val) -{ - val = XXH64_round(0, val); - acc ^= val; - acc = acc * PRIME64_1 + PRIME64_4; - return acc; -} - -static U64 XXH64_avalanche(U64 h64) -{ - h64 ^= h64 >> 33; - h64 *= PRIME64_2; - h64 ^= h64 >> 29; - h64 *= PRIME64_3; - h64 ^= h64 >> 32; - return h64; -} - - -#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align) - -static U64 -XXH64_finalize(U64 h64, const void* ptr, size_t len, - XXH_endianess endian, XXH_alignment align) -{ - const BYTE* p = (const BYTE*)ptr; - -#define PROCESS1_64 \ - h64 ^= (*p++) * PRIME64_5; \ - h64 = XXH_rotl64(h64, 11) * PRIME64_1; - -#define PROCESS4_64 \ - h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; \ - p+=4; \ - h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; - -#define PROCESS8_64 { \ - U64 const k1 = XXH64_round(0, XXH_get64bits(p)); \ - p+=8; \ - h64 ^= k1; \ - h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; \ -} - - switch(len&31) { - case 24: PROCESS8_64; - /* fallthrough */ - case 16: PROCESS8_64; - /* fallthrough */ - case 8: PROCESS8_64; - return XXH64_avalanche(h64); - - case 28: PROCESS8_64; - /* fallthrough */ - case 20: PROCESS8_64; - /* fallthrough */ - case 12: PROCESS8_64; - /* fallthrough */ - case 4: PROCESS4_64; - return XXH64_avalanche(h64); - - case 25: PROCESS8_64; - /* fallthrough */ - case 17: PROCESS8_64; - /* fallthrough */ - case 9: PROCESS8_64; - PROCESS1_64; - return XXH64_avalanche(h64); - - case 29: PROCESS8_64; - /* fallthrough */ - case 21: PROCESS8_64; - /* fallthrough */ - case 13: PROCESS8_64; - /* fallthrough */ - case 5: PROCESS4_64; - PROCESS1_64; - return XXH64_avalanche(h64); - - case 26: PROCESS8_64; - /* fallthrough */ - case 18: PROCESS8_64; - /* fallthrough */ - case 10: PROCESS8_64; - PROCESS1_64; - PROCESS1_64; - return XXH64_avalanche(h64); - - case 30: PROCESS8_64; - /* fallthrough */ - case 22: PROCESS8_64; - /* fallthrough */ - case 14: PROCESS8_64; - /* fallthrough */ - case 6: PROCESS4_64; - PROCESS1_64; - PROCESS1_64; - return XXH64_avalanche(h64); - - case 27: PROCESS8_64; - /* fallthrough */ - case 19: PROCESS8_64; - /* fallthrough */ - case 11: PROCESS8_64; - PROCESS1_64; - PROCESS1_64; - PROCESS1_64; - return XXH64_avalanche(h64); - - case 31: PROCESS8_64; - /* fallthrough */ - case 23: PROCESS8_64; - /* fallthrough */ - case 15: PROCESS8_64; - /* fallthrough */ - case 7: PROCESS4_64; - /* fallthrough */ - case 3: PROCESS1_64; - /* fallthrough */ - case 2: PROCESS1_64; - /* fallthrough */ - case 1: PROCESS1_64; - /* fallthrough */ - case 0: return XXH64_avalanche(h64); - } - - /* impossible to reach */ - assert(0); - return 0; /* unreachable, but some compilers complain without it */ -} - -FORCE_INLINE U64 -XXH64_endian_align(const void* input, size_t len, U64 seed, - XXH_endianess endian, XXH_alignment align) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* bEnd = p + len; - U64 h64; - -#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) - if (p==NULL) { - len=0; - bEnd=p=(const BYTE*)(size_t)32; - } -#endif - - if (len>=32) { - const BYTE* const limit = bEnd - 32; - U64 v1 = seed + PRIME64_1 + PRIME64_2; - U64 v2 = seed + PRIME64_2; - U64 v3 = seed + 0; - U64 v4 = seed - PRIME64_1; - - do { - v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8; - v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8; - v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8; - v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8; - } while (p<=limit); - - h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); - h64 = XXH64_mergeRound(h64, v1); - h64 = XXH64_mergeRound(h64, v2); - h64 = XXH64_mergeRound(h64, v3); - h64 = XXH64_mergeRound(h64, v4); - - } else { - h64 = seed + PRIME64_5; - } - - h64 += (U64) len; - - return XXH64_finalize(h64, p, len, endian, align); -} - - -XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed) -{ -#if 0 - /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ - XXH64_state_t state; - XXH64_reset(&state, seed); - XXH64_update(&state, input, len); - return XXH64_digest(&state); -#else - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if (XXH_FORCE_ALIGN_CHECK) { - if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); - else - return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); - } } - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); - else - return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); -#endif -} - -/*====== Hash Streaming ======*/ - -XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) -{ - return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); -} -XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) -{ - XXH_free(statePtr); - return XXH_OK; -} - -XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dstState, const XXH64_state_t* srcState) -{ - memcpy(dstState, srcState, sizeof(*dstState)); -} - -XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed) -{ - XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ - memset(&state, 0, sizeof(state)); - state.v1 = seed + PRIME64_1 + PRIME64_2; - state.v2 = seed + PRIME64_2; - state.v3 = seed + 0; - state.v4 = seed - PRIME64_1; - /* do not write into reserved, planned to be removed in a future version */ - memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved)); - return XXH_OK; -} - -FORCE_INLINE XXH_errorcode -XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian) -{ - if (input==NULL) -#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) - return XXH_OK; -#else - return XXH_ERROR; -#endif - - { const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - - state->total_len += len; - - if (state->memsize + len < 32) { /* fill in tmp buffer */ - XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); - state->memsize += (U32)len; - return XXH_OK; - } - - if (state->memsize) { /* tmp buffer is full */ - XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize); - state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian)); - state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian)); - state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian)); - state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian)); - p += 32-state->memsize; - state->memsize = 0; - } - - if (p+32 <= bEnd) { - const BYTE* const limit = bEnd - 32; - U64 v1 = state->v1; - U64 v2 = state->v2; - U64 v3 = state->v3; - U64 v4 = state->v4; - - do { - v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8; - v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8; - v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8; - v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8; - } while (p<=limit); - - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; - } - - if (p < bEnd) { - XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); - state->memsize = (unsigned)(bEnd-p); - } - } - - return XXH_OK; -} - -XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_update_endian(state_in, input, len, XXH_littleEndian); - else - return XXH64_update_endian(state_in, input, len, XXH_bigEndian); -} - -FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian) -{ - U64 h64; - - if (state->total_len >= 32) { - U64 const v1 = state->v1; - U64 const v2 = state->v2; - U64 const v3 = state->v3; - U64 const v4 = state->v4; - - h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); - h64 = XXH64_mergeRound(h64, v1); - h64 = XXH64_mergeRound(h64, v2); - h64 = XXH64_mergeRound(h64, v3); - h64 = XXH64_mergeRound(h64, v4); - } else { - h64 = state->v3 /*seed*/ + PRIME64_5; - } - - h64 += (U64) state->total_len; - - return XXH64_finalize(h64, state->mem64, (size_t)state->total_len, endian, XXH_aligned); -} - -XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_digest_endian(state_in, XXH_littleEndian); - else - return XXH64_digest_endian(state_in, XXH_bigEndian); -} - - -/*====== Canonical representation ======*/ - -XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash) -{ - XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); - if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); - memcpy(dst, &hash, sizeof(*dst)); -} - -XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src) -{ - return XXH_readBE64(src); -} - -#endif /* XXH_NO_LONG_LONG */ diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 63760d6ae4..a24efeac5f 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -181,6 +181,7 @@ ,,y,script,./test.sh -f tsim/query/groupby.sim ,,y,script,./test.sh -f tsim/query/event.sim ,,y,script,./test.sh -f tsim/query/forceFill.sim +,,n,script,./test.sh -f tsim/query/join.sim ,,y,script,./test.sh -f tsim/qnode/basic1.sim ,,y,script,./test.sh -f tsim/snode/basic1.sim ,,y,script,./test.sh -f tsim/mnode/basic1.sim @@ -651,6 +652,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/elapsed.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/csum.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_diff.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tagFilter.py ,,n,system-test,python3 ./test.py -f 2-query/queryQnode.py ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode1mnode.py ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 diff --git a/tests/script/sh/checkAsan.sh b/tests/script/sh/checkAsan.sh index 20d1359da8..a793ef4436 100755 --- a/tests/script/sh/checkAsan.sh +++ b/tests/script/sh/checkAsan.sh @@ -50,6 +50,7 @@ python_error=`cat ${LOG_DIR}/*.info | grep -w "stack" | wc -l` # /root/TDengine/source/libs/function/src/builtinsimpl.c:856:29: runtime error: signed integer overflow: 9223372036854775806 + 9223372036854775805 cannot be represented in type 'long int' # /root/TDengine/source/libs/scalar/src/sclvector.c:1075:66: runtime error: signed integer overflow: 9223372034707292160 + 1668838476672 cannot be represented in type 'long int' # /root/TDengine/source/common/src/tdataformat.c:1876:7: runtime error: signed integer overflow: 8252423483843671206 + 2406154664059062870 cannot be represented in type 'long int' +# /home/chr/TDengine/source/libs/scalar/src/filter.c:3149:14: runtime error: applying non-zero offset 18446744073709551615 to null pointer # /home/chr/TDengine/source/libs/scalar/src/filter.c:3149:14: runtime error: applying non-zero offset 18446744073709551615 to null pointer # /home/TDinternal/community/source/libs/scalar/src/sclvector.c:1109:66: runtime error: signed integer overflow: 9223372034707292160 + 1676867897049 cannot be represented in type 'long int' diff --git a/tests/script/tsim/query/join.sim b/tests/script/tsim/query/join.sim new file mode 100644 index 0000000000..adb0338ef7 --- /dev/null +++ b/tests/script/tsim/query/join.sim @@ -0,0 +1,72 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +$dbPrefix = db +$tbPrefix1 = tba +$tbPrefix2 = tbb +$mtPrefix = stb +$tbNum = 10000 +$rowNum = 2 + +print =============== step1 +$i = 0 +$db = $dbPrefix . $i +$mt1 = $mtPrefix . $i +$i = 1 +$mt2 = $mtPrefix . $i + +sql drop database $db -x step1 +step1: +sql create database $db +sql use $db +sql create table $mt1 (ts timestamp, f1 int) TAGS(tag1 int, tag2 binary(500)) +sql create table $mt2 (ts timestamp, f1 int) TAGS(tag1 int, tag2 binary(500)) + +print ====== start create child tables and insert data +$i = 0 +while $i < $tbNum + $tb = $tbPrefix1 . $i + sql create table $tb using $mt1 tags( $i , 'aaaaaaaaaaaaaaaaaaaaaaaaaaa') + + $x = 0 + while $x < $rowNum + $cc = $x * 60000 + $ms = 1601481600000 + $cc + + sql insert into $tb values ($ms , $x ) + $x = $x + 1 + endw + + $i = $i + 1 +endw + +print =============== step2 +$i = 0 +while $i < $tbNum + $tb = $tbPrefix2 . $i + sql create table $tb using $mt2 tags( $i , 'aaaaaaaaaaaaaaaaaaaaaaaaaaa') + + $x = 0 + while $x < $rowNum + $cc = $x * 60000 + $ms = 1601481600000 + $cc + + sql insert into $tb values ($ms , $x ) + $x = $x + 1 + endw + + $i = $i + 1 +endw + +sql select * from tba0 t1, tbb0 t2 where t1.ts=t2.ts; +if $rows != 2 then + return -1 +endi +sql select * from stb0 t1, stb1 t2 where t1.ts=t2.ts and t1.tag2=t2.tag2; +if $rows != 200000000 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/system-test/2-query/tagFilter.py b/tests/system-test/2-query/tagFilter.py new file mode 100644 index 0000000000..b03776c31b --- /dev/null +++ b/tests/system-test/2-query/tagFilter.py @@ -0,0 +1,67 @@ +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + + +INT_COL = "c1" +BINT_COL = "c2" +SINT_COL = "c3" +TINT_COL = "c4" +FLOAT_COL = "c5" +DOUBLE_COL = "c6" +BOOL_COL = "c7" + +BINARY_COL = "c8" +NCHAR_COL = "c9" +TS_COL = "c10" + +NUM_COL = [INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ] +UN_NUM_COL = [BOOL_COL, BINARY_COL, NCHAR_COL, ] +TS_TYPE_COL = [TS_COL] + +DBNAME = "db" + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + tdSql.execute(f'drop database if exists db') + tdSql.execute(f'create database if not exists db vgroups 1') + + def __create_tb(self, dbname="db"): + create_stb_sql = f'''create table {dbname}.stb1( + ts timestamp, f1 int + ) tags (tag1 binary(16300)) + ''' + tdSql.execute(create_stb_sql) + + tag_value = 'a' + for i in range(1200): + tag_value = tag_value + 'a' + + for i in range(8000): + tdSql.execute(f"create table {dbname}.ct{i+1} using {dbname}.stb1 tags ( '{tag_value}' )") + + def __query_data(self, rows, dbname="db"): + tdSql.execute( + f'''select count(*) from {dbname}.stb1 where tag1 like '%a' + ''' + ) + tdSql.checkRows(0) + + def run(self): + tdLog.printNoPrefix("==========step1:create table") + self.__create_tb() + + tdLog.printNoPrefix("==========step2:query data") + self.__query_data(10) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/utils/test/c/tmqSim.c b/utils/test/c/tmqSim.c index 9c1dc2e063..1acf50a7d8 100644 --- a/utils/test/c/tmqSim.c +++ b/utils/test/c/tmqSim.c @@ -400,7 +400,7 @@ TAOS* createNewTaosConnect() { int32_t retryCnt = 10; while (retryCnt--) { - TAOS* taos = taos_connect(NULL, "root", "taosdata", NULL, 0); + taos = taos_connect(NULL, "root", "taosdata", NULL, 0); if (NULL != taos) { return taos; } @@ -780,7 +780,8 @@ void loop_consume(SThreadInfo* pInfo) { if (pInfo->ifCheckData) { char filename[256] = {0}; - char tmpString[128]; + memset(tmpString, 0, tListLen(tmpString)); + // sprintf(filename, "%s/../log/consumerid_%d_%s.txt", configDir, pInfo->consumerId, // getCurrentTimeString(tmpString)); sprintf(filename, "%s/../log/consumerid_%d.txt", configDir, pInfo->consumerId); @@ -834,12 +835,12 @@ void loop_consume(SThreadInfo* pInfo) { } if ((totalRows >= pInfo->expectMsgCnt) || (totalMsgs >= pInfo->expectMsgCnt)) { - char tmpString[128]; + memset(tmpString, 0, tListLen(tmpString)); taosFprintfFile(g_fp, "%s over than expect rows, so break consume\n", getCurrentTimeString(tmpString)); break; } } else { - char tmpString[128]; + memset(tmpString, 0, tListLen(tmpString)); taosFprintfFile(g_fp, "%s no poll more msg when time over, break consume\n", getCurrentTimeString(tmpString)); break; } @@ -1113,7 +1114,7 @@ void omb_loop_consume(SThreadInfo* pInfo) { lastTotalLenOfMsg = totalLenOfMsg; } } else { - char tmpString[128]; + memset(tmpString, 0, tListLen(tmpString)); taosFprintfFile(g_fp, "%s no poll more msg when time over, break consume\n", getCurrentTimeString(tmpString)); printf("%s no poll more msg when time over, break consume\n", getCurrentTimeString(tmpString)); int64_t currentPrintTime = taosGetTimestampMs(); @@ -1381,7 +1382,7 @@ void startOmbConsume() { printf("SQL: %s\n", sql); queryDbExec(taos, sql, NO_INSERT_TYPE); - int32_t producerRate = ceil(g_stConfInfo.producerRate / g_stConfInfo.producers); + int32_t producerRate = ceil(((double)g_stConfInfo.producerRate) / g_stConfInfo.producers); printf("==== create %d produce thread ====\n", g_stConfInfo.producers); for (int32_t i = 0; i < g_stConfInfo.producers; ++i) {