diff --git a/Jenkinsfile2 b/Jenkinsfile2 index d827c6640a..d90bc3061b 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -47,36 +47,67 @@ def pre_test(){ script { if (env.CHANGE_TARGET == 'master') { sh ''' + cd ${WK} + git checkout master cd ${WKC} git checkout master ''' } else if(env.CHANGE_TARGET == '2.0') { sh ''' + cd ${WK} + git checkout 2.0 cd ${WKC} git checkout 2.0 ''' } else if(env.CHANGE_TARGET == '3.0') { sh ''' + cd ${WK} + git checkout 3.0 cd ${WKC} git checkout 3.0 - [ -d contrib/bdb ] && cd contrib/bdb && git clean -fxd && cd ../.. ''' } else { sh ''' + cd ${WK} + git checkout develop cd ${WKC} git checkout develop ''' } } + if (env.CHANGE_URL =~ /\/TDengine\//) { + sh ''' + cd ${WKC} + git pull >/dev/null + git fetch origin +refs/pull/${CHANGE_ID}/merge + git checkout -qf FETCH_HEAD + git log|head -n20 + cd ${WK} + git pull >/dev/null + git log|head -n20 + ''' + } else if (env.CHANGE_URL =~ /\/TDinternal\//) { + sh ''' + cd ${WK} + git pull >/dev/null + git fetch origin +refs/pull/${CHANGE_ID}/merge + git checkout -qf FETCH_HEAD + git log|head -n20 + cd ${WKC} + git pull >/dev/null + git log|head -n20 + ''' + } else { + sh ''' + echo "unmatched reposiotry ${CHANGE_URL}" + ''' + } sh ''' cd ${WKC} - git pull >/dev/null - git fetch origin +refs/pull/${CHANGE_ID}/merge - git checkout -qf FETCH_HEAD git submodule update --init --recursive ''' sh ''' - cd ${WKC} + cd ${WK} export TZ=Asia/Harbin date rm -rf debug @@ -162,8 +193,8 @@ pipeline { options { skipDefaultCheckout() } environment{ WK = '/var/lib/jenkins/workspace/TDinternal' - WKC= '/var/lib/jenkins/workspace/TDengine' - WKPY= '/var/lib/jenkins/workspace/taos-connector-python' + WKC = '/var/lib/jenkins/workspace/TDinternal/community' + WKPY = '/var/lib/jenkins/workspace/taos-connector-python' } stages { stage('run test') { @@ -177,15 +208,33 @@ pipeline { steps { timeout(time: 45, unit: 'MINUTES'){ pre_test() - sh ''' - cd ${WKC}/debug - ctest -VV - ''' - sh ''' - export LD_LIBRARY_PATH=${WKC}/debug/build/lib - cd ${WKC}/tests/system-test - ./fulltest.sh - ''' + script { + if (env.CHANGE_URL =~ /\/TDengine\//) { + sh ''' + cd ${WK}/debug + ctest -VV + ''' + sh ''' + export LD_LIBRARY_PATH=${WK}/debug/build/lib + cd ${WKC}/tests/system-test + ./fulltest.sh + ''' + } else if (env.CHANGE_URL =~ /\/TDinternal\//) { + sh ''' + cd ${WKC}/debug + ctest -VV + ''' + sh ''' + export LD_LIBRARY_PATH=${WKC}/debug/build/lib + cd ${WKC}/tests/system-test + ./fulltest.sh + ''' + } else { + sh ''' + echo "unmatched reposiotry ${CHANGE_URL}" + ''' + } + } sh ''' cd ${WKC}/tests ./test-all.sh b1fq diff --git a/cmake/cmake.define b/cmake/cmake.define index f90024e032..0eb5206aab 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -85,11 +85,11 @@ ELSE () IF (${SANITIZER} MATCHES "true") SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3") MESSAGE(STATUS "Will compile with Address Sanitizer!") ELSE () SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-literal-suffix -Werror=return-type -fpermissive -fPIC -gdwarf-2 -g3") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -g3") ENDIF () MESSAGE("System processor ID: ${CMAKE_SYSTEM_PROCESSOR}") diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 1f3b787538..2e8a214a9d 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -18,6 +18,7 @@ #include "os.h" #include "talgo.h" +#include "tencode.h" #include "ttypes.h" #include "tutil.h" @@ -25,6 +26,73 @@ extern "C" { #endif +typedef struct SSchema SSchema; +typedef struct STColumn STColumn; +typedef struct STSchema STSchema; +typedef struct STSRow2 STSRow2; +typedef struct STSRowBuilder STSRowBuilder; +typedef struct SKVIdx SKVIdx; + +// STSchema + +// STSRow2 +int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow); +int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow); + +// STSchema +int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema); +void tTSchemaDestroy(STSchema *pTSchema); + +// STSRowBuilder +int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchema, int32_t nCols); +void tTSRowBuilderClear(STSRowBuilder *pBuilder); +void tTSRowBuilderReset(STSRowBuilder *pBuilder); +int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pData, uint32_t nData); +int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow); + +// STRUCT ================= +struct STColumn { + col_id_t colId; + int8_t type; + int8_t flags; + int32_t bytes; + int32_t offset; +}; + +struct STSchema { + int32_t numOfCols; + int32_t version; + int32_t flen; + int32_t vlen; + int32_t tlen; + STColumn columns[]; +}; + +struct STSRow2 { + TSKEY ts; + uint32_t flags; + union { + int32_t sver; + int32_t ncols; + }; + uint32_t nData; + const uint8_t *pData; +}; + +struct STSRowBuilder { + STColumn *pTColumn; + STSchema *pTSchema; + int32_t szKVBuf; + uint8_t *pKVBuf; + int32_t szTPBuf; + uint8_t *pTPBuf; + int32_t nCols; + int32_t kvVLen; + int32_t tpVLen; + STSRow2 row; +}; + +#if 1 //==================================== // Imported since 3.0 and use bitmap to demonstrate None/Null/Norm, while use Null/Norm below 3.0 without of bitmap. #define TD_SUPPORT_BITMAP #define TD_SUPPORT_READ2 @@ -59,15 +127,6 @@ extern "C" { } while (0); // ----------------- TSDB COLUMN DEFINITION -#pragma pack(push, 1) -typedef struct { - col_id_t colId; // column ID(start from PRIMARYKEY_TIMESTAMP_COL_ID(1)) - int8_t type; // column type - int8_t flags; // flags: 0 no index, 1 SCHEMA_SMA_ON, 2 SCHEMA_IDX_ON - int32_t bytes; // column bytes (0~16M) - int32_t offset; // point offset in STpRow after the header part. -} STColumn; -#pragma pack(pop) #define colType(col) ((col)->type) #define colFlags(col) ((col)->flags) @@ -82,15 +141,6 @@ typedef struct { #define colSetOffset(col, o) (colOffset(col) = (o)) // ----------------- TSDB SCHEMA DEFINITION -typedef struct { - int32_t numOfCols; // Number of columns appended - schema_ver_t version; // schema version - uint16_t flen; // First part length in a STpRow after the header part - int32_t vlen; // pure value part length, excluded the overhead (bytes only) - int32_t tlen; // maximum length of a STpRow without the header part - // (sizeof(VarDataOffsetT) + sizeof(VarDataLenT) + (bytes)) - STColumn columns[]; -} STSchema; #define schemaNCols(s) ((s)->numOfCols) #define schemaVersion(s) ((s)->version) @@ -386,6 +436,7 @@ static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t co return 0; } +#endif #ifdef __cplusplus } diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 953211c491..7f17c1673b 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -273,8 +273,10 @@ int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp); int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp); void tFreeSSubmitRsp(SSubmitRsp *pRsp); -#define SCHEMA_SMA_ON 0x1 -#define SCHEMA_IDX_ON 0x2 +#define COL_SMA_ON ((int8_t)0x1) +#define COL_IDX_ON ((int8_t)0x2) +#define COL_VAL_SET ((int8_t)0x4) + typedef struct SSchema { int8_t type; int8_t flags; @@ -283,7 +285,7 @@ typedef struct SSchema { char name[TSDB_COL_NAME_LEN]; } SSchema; -#define IS_BSMA_ON(s) (((s)->flags & 0x01) == SCHEMA_SMA_ON) +#define IS_BSMA_ON(s) (((s)->flags & 0x01) == COL_SMA_ON) #define SSCHMEA_TYPE(s) ((s)->type) #define SSCHMEA_FLAGS(s) ((s)->flags) @@ -297,6 +299,109 @@ typedef struct { SSchema* pSchema; } SSchemaWrapper; +static FORCE_INLINE SSchemaWrapper* tCloneSSchemaWrapper(const SSchemaWrapper* pSchemaWrapper) { + SSchemaWrapper* pSW = (SSchemaWrapper*)taosMemoryMalloc(sizeof(SSchemaWrapper)); + if (pSW == NULL) return pSW; + pSW->nCols = pSchemaWrapper->nCols; + pSW->sver = pSchemaWrapper->sver; + pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema)); + if (pSW->pSchema == NULL) { + taosMemoryFree(pSW); + return NULL; + } + memcpy(pSW->pSchema, pSchemaWrapper->pSchema, pSW->nCols * sizeof(SSchema)); + return pSW; +} + +static FORCE_INLINE void tDeleteSSchemaWrapper(SSchemaWrapper* pSchemaWrapper) { + taosMemoryFree(pSchemaWrapper->pSchema); + taosMemoryFree(pSchemaWrapper); +} + +static FORCE_INLINE int32_t taosEncodeSSchema(void** buf, const SSchema* pSchema) { + int32_t tlen = 0; + tlen += taosEncodeFixedI8(buf, pSchema->type); + tlen += taosEncodeFixedI8(buf, pSchema->flags); + tlen += taosEncodeFixedI32(buf, pSchema->bytes); + tlen += taosEncodeFixedI16(buf, pSchema->colId); + tlen += taosEncodeString(buf, pSchema->name); + return tlen; +} + +static FORCE_INLINE void* taosDecodeSSchema(const void* buf, SSchema* pSchema) { + buf = taosDecodeFixedI8(buf, &pSchema->type); + buf = taosDecodeFixedI8(buf, &pSchema->flags); + buf = taosDecodeFixedI32(buf, &pSchema->bytes); + buf = taosDecodeFixedI16(buf, &pSchema->colId); + buf = taosDecodeStringTo(buf, pSchema->name); + return (void*)buf; +} + +static FORCE_INLINE int32_t tEncodeSSchema(SEncoder* pEncoder, const SSchema* pSchema) { + if (tEncodeI8(pEncoder, pSchema->type) < 0) return -1; + if (tEncodeI8(pEncoder, pSchema->flags) < 0) return -1; + if (tEncodeI32v(pEncoder, pSchema->bytes) < 0) return -1; + if (tEncodeI16v(pEncoder, pSchema->colId) < 0) return -1; + if (tEncodeCStr(pEncoder, pSchema->name) < 0) return -1; + return 0; +} + +static FORCE_INLINE int32_t tDecodeSSchema(SDecoder* pDecoder, SSchema* pSchema) { + if (tDecodeI8(pDecoder, &pSchema->type) < 0) return -1; + if (tDecodeI8(pDecoder, &pSchema->flags) < 0) return -1; + if (tDecodeI32v(pDecoder, &pSchema->bytes) < 0) return -1; + if (tDecodeI16v(pDecoder, &pSchema->colId) < 0) return -1; + if (tDecodeCStrTo(pDecoder, pSchema->name) < 0) return -1; + return 0; +} + +static FORCE_INLINE int32_t taosEncodeSSchemaWrapper(void** buf, const SSchemaWrapper* pSW) { + int32_t tlen = 0; + tlen += taosEncodeVariantI32(buf, pSW->nCols); + tlen += taosEncodeVariantI32(buf, pSW->sver); + for (int32_t i = 0; i < pSW->nCols; i++) { + tlen += taosEncodeSSchema(buf, &pSW->pSchema[i]); + } + return tlen; +} + +static FORCE_INLINE void* taosDecodeSSchemaWrapper(const void* buf, SSchemaWrapper* pSW) { + buf = taosDecodeVariantI32(buf, &pSW->nCols); + buf = taosDecodeVariantI32(buf, &pSW->sver); + pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema)); + if (pSW->pSchema == NULL) { + return NULL; + } + + for (int32_t i = 0; i < pSW->nCols; i++) { + buf = taosDecodeSSchema(buf, &pSW->pSchema[i]); + } + return (void*)buf; +} + +static FORCE_INLINE int32_t tEncodeSSchemaWrapper(SEncoder* pEncoder, const SSchemaWrapper* pSW) { + if (tEncodeI32v(pEncoder, pSW->nCols) < 0) return -1; + if (tEncodeI32v(pEncoder, pSW->sver) < 0) return -1; + for (int32_t i = 0; i < pSW->nCols; i++) { + if (tEncodeSSchema(pEncoder, &pSW->pSchema[i]) < 0) return -1; + } + + return 0; +} + +static FORCE_INLINE int32_t tDecodeSSchemaWrapper(SDecoder* pDecoder, SSchemaWrapper* pSW) { + if (tDecodeI32v(pDecoder, &pSW->nCols) < 0) return -1; + if (tDecodeI32v(pDecoder, &pSW->sver) < 0) return -1; + + pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema)); + if (pSW->pSchema == NULL) return -1; + for (int32_t i = 0; i < pSW->nCols; i++) { + if (tDecodeSSchema(pDecoder, &pSW->pSchema[i]) < 0) return -1; + } + + return 0; +} + STSchema* tdGetSTSChemaFromSSChema(SSchema** pSchema, int32_t nCols); typedef struct { @@ -1880,7 +1985,7 @@ typedef struct SMqHbTopicInfo { int32_t epoch; int64_t topicUid; char name[TSDB_TOPIC_FNAME_LEN]; - SArray* pVgInfo; + SArray* pVgInfo; // SArray } SMqHbTopicInfo; static FORCE_INLINE int32_t taosEncodeSMqHbTopicInfoMsg(void** buf, const SMqHbTopicInfo* pTopicInfo) { @@ -1999,49 +2104,6 @@ static FORCE_INLINE void* tDecodeSMqRebVgReq(const void* buf, SMqRebVgReq* pReq) return (void*)buf; } -typedef struct { - int8_t reserved; -} SMqRebVgRsp; - -typedef struct { - int64_t leftForVer; - int32_t vgId; - int32_t epoch; - int64_t consumerId; - char topicName[TSDB_TOPIC_FNAME_LEN]; - char cgroup[TSDB_CGROUP_LEN]; - char* sql; - char* physicalPlan; - char* qmsg; -} SMqSetCVgReq; - -static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq* pReq) { - int32_t tlen = 0; - tlen += taosEncodeFixedI64(buf, pReq->leftForVer); - tlen += taosEncodeFixedI32(buf, pReq->vgId); - tlen += taosEncodeFixedI32(buf, pReq->epoch); - tlen += taosEncodeFixedI64(buf, pReq->consumerId); - tlen += taosEncodeString(buf, pReq->topicName); - tlen += taosEncodeString(buf, pReq->cgroup); - tlen += taosEncodeString(buf, pReq->sql); - tlen += taosEncodeString(buf, pReq->physicalPlan); - tlen += taosEncodeString(buf, pReq->qmsg); - return tlen; -} - -static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) { - buf = taosDecodeFixedI64(buf, &pReq->leftForVer); - buf = taosDecodeFixedI32(buf, &pReq->vgId); - buf = taosDecodeFixedI32(buf, &pReq->epoch); - buf = taosDecodeFixedI64(buf, &pReq->consumerId); - buf = taosDecodeStringTo(buf, pReq->topicName); - buf = taosDecodeStringTo(buf, pReq->cgroup); - buf = taosDecodeString(buf, &pReq->sql); - buf = taosDecodeString(buf, &pReq->physicalPlan); - buf = taosDecodeString(buf, &pReq->qmsg); - return buf; -} - typedef struct { int32_t vgId; int64_t offset; @@ -2063,109 +2125,6 @@ int32_t tDecodeSMqOffset(SDecoder* decoder, SMqOffset* pOffset); int32_t tEncodeSMqCMCommitOffsetReq(SEncoder* encoder, const SMqCMCommitOffsetReq* pReq); int32_t tDecodeSMqCMCommitOffsetReq(SDecoder* decoder, SMqCMCommitOffsetReq* pReq); -static FORCE_INLINE SSchemaWrapper* tCloneSSchemaWrapper(const SSchemaWrapper* pSchemaWrapper) { - SSchemaWrapper* pSW = (SSchemaWrapper*)taosMemoryMalloc(sizeof(SSchemaWrapper)); - if (pSW == NULL) return pSW; - pSW->nCols = pSchemaWrapper->nCols; - pSW->sver = pSchemaWrapper->sver; - pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema)); - if (pSW->pSchema == NULL) { - taosMemoryFree(pSW); - return NULL; - } - memcpy(pSW->pSchema, pSchemaWrapper->pSchema, pSW->nCols * sizeof(SSchema)); - return pSW; -} - -static FORCE_INLINE void tDeleteSSchemaWrapper(SSchemaWrapper* pSchemaWrapper) { - taosMemoryFree(pSchemaWrapper->pSchema); - taosMemoryFree(pSchemaWrapper); -} - -static FORCE_INLINE int32_t taosEncodeSSchema(void** buf, const SSchema* pSchema) { - int32_t tlen = 0; - tlen += taosEncodeFixedI8(buf, pSchema->type); - tlen += taosEncodeFixedI8(buf, pSchema->flags); - tlen += taosEncodeFixedI32(buf, pSchema->bytes); - tlen += taosEncodeFixedI16(buf, pSchema->colId); - tlen += taosEncodeString(buf, pSchema->name); - return tlen; -} - -static FORCE_INLINE void* taosDecodeSSchema(const void* buf, SSchema* pSchema) { - buf = taosDecodeFixedI8(buf, &pSchema->type); - buf = taosDecodeFixedI8(buf, &pSchema->flags); - buf = taosDecodeFixedI32(buf, &pSchema->bytes); - buf = taosDecodeFixedI16(buf, &pSchema->colId); - buf = taosDecodeStringTo(buf, pSchema->name); - return (void*)buf; -} - -static FORCE_INLINE int32_t tEncodeSSchema(SEncoder* pEncoder, const SSchema* pSchema) { - if (tEncodeI8(pEncoder, pSchema->type) < 0) return -1; - if (tEncodeI8(pEncoder, pSchema->flags) < 0) return -1; - if (tEncodeI32v(pEncoder, pSchema->bytes) < 0) return -1; - if (tEncodeI16v(pEncoder, pSchema->colId) < 0) return -1; - if (tEncodeCStr(pEncoder, pSchema->name) < 0) return -1; - return 0; -} - -static FORCE_INLINE int32_t tDecodeSSchema(SDecoder* pDecoder, SSchema* pSchema) { - if (tDecodeI8(pDecoder, &pSchema->type) < 0) return -1; - if (tDecodeI8(pDecoder, &pSchema->flags) < 0) return -1; - if (tDecodeI32v(pDecoder, &pSchema->bytes) < 0) return -1; - if (tDecodeI16v(pDecoder, &pSchema->colId) < 0) return -1; - if (tDecodeCStrTo(pDecoder, pSchema->name) < 0) return -1; - return 0; -} - -static FORCE_INLINE int32_t taosEncodeSSchemaWrapper(void** buf, const SSchemaWrapper* pSW) { - int32_t tlen = 0; - tlen += taosEncodeVariantI32(buf, pSW->nCols); - tlen += taosEncodeVariantI32(buf, pSW->sver); - for (int32_t i = 0; i < pSW->nCols; i++) { - tlen += taosEncodeSSchema(buf, &pSW->pSchema[i]); - } - return tlen; -} - -static FORCE_INLINE void* taosDecodeSSchemaWrapper(const void* buf, SSchemaWrapper* pSW) { - buf = taosDecodeVariantI32(buf, &pSW->nCols); - buf = taosDecodeVariantI32(buf, &pSW->sver); - pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema)); - if (pSW->pSchema == NULL) { - return NULL; - } - - for (int32_t i = 0; i < pSW->nCols; i++) { - buf = taosDecodeSSchema(buf, &pSW->pSchema[i]); - } - return (void*)buf; -} - -static FORCE_INLINE int32_t tEncodeSSchemaWrapper(SEncoder* pEncoder, const SSchemaWrapper* pSW) { - if (tEncodeI32v(pEncoder, pSW->nCols) < 0) return -1; - if (tEncodeI32v(pEncoder, pSW->sver) < 0) return -1; - for (int32_t i = 0; i < pSW->nCols; i++) { - if (tEncodeSSchema(pEncoder, &pSW->pSchema[i]) < 0) return -1; - } - - return 0; -} - -static FORCE_INLINE int32_t tDecodeSSchemaWrapper(SDecoder* pDecoder, SSchemaWrapper* pSW) { - if (tDecodeI32v(pDecoder, &pSW->nCols) < 0) return -1; - if (tDecodeI32v(pDecoder, &pSW->sver) < 0) return -1; - - pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema)); - if (pSW->pSchema == NULL) return -1; - for (int32_t i = 0; i < pSW->nCols; i++) { - if (tDecodeSSchema(pDecoder, &pSW->pSchema[i]) < 0) return -1; - } - - return 0; -} - typedef struct { char name[TSDB_TABLE_FNAME_LEN]; char stb[TSDB_TABLE_FNAME_LEN]; @@ -2434,6 +2393,21 @@ typedef struct { SEpSet epSet; } SMqSubVgEp; +static FORCE_INLINE int32_t tEncodeSMqSubVgEp(void** buf, const SMqSubVgEp* pVgEp) { + int32_t tlen = 0; + tlen += taosEncodeFixedI32(buf, pVgEp->vgId); + tlen += taosEncodeFixedI64(buf, pVgEp->offset); + tlen += taosEncodeSEpSet(buf, &pVgEp->epSet); + return tlen; +} + +static FORCE_INLINE void* tDecodeSMqSubVgEp(void* buf, SMqSubVgEp* pVgEp) { + buf = taosDecodeFixedI32(buf, &pVgEp->vgId); + buf = taosDecodeFixedI64(buf, &pVgEp->offset); + buf = taosDecodeSEpSet(buf, &pVgEp->epSet); + return buf; +} + typedef struct { char topic[TSDB_TOPIC_FNAME_LEN]; int8_t isSchemaAdaptive; @@ -2441,6 +2415,43 @@ typedef struct { SSchemaWrapper schema; } SMqSubTopicEp; +static FORCE_INLINE int32_t tEncodeSMqSubTopicEp(void** buf, const SMqSubTopicEp* pTopicEp) { + int32_t tlen = 0; + tlen += taosEncodeString(buf, pTopicEp->topic); + tlen += taosEncodeFixedI8(buf, pTopicEp->isSchemaAdaptive); + int32_t sz = taosArrayGetSize(pTopicEp->vgs); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + SMqSubVgEp* pVgEp = (SMqSubVgEp*)taosArrayGet(pTopicEp->vgs, i); + tlen += tEncodeSMqSubVgEp(buf, pVgEp); + } + tlen += taosEncodeSSchemaWrapper(buf, &pTopicEp->schema); + return tlen; +} + +static FORCE_INLINE void* tDecodeSMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicEp) { + buf = taosDecodeStringTo(buf, pTopicEp->topic); + buf = taosDecodeFixedI8(buf, &pTopicEp->isSchemaAdaptive); + int32_t sz; + buf = taosDecodeFixedI32(buf, &sz); + pTopicEp->vgs = taosArrayInit(sz, sizeof(SMqSubVgEp)); + if (pTopicEp->vgs == NULL) { + return NULL; + } + for (int32_t i = 0; i < sz; i++) { + SMqSubVgEp vgEp; + buf = tDecodeSMqSubVgEp(buf, &vgEp); + taosArrayPush(pTopicEp->vgs, &vgEp); + } + buf = taosDecodeSSchemaWrapper(buf, &pTopicEp->schema); + return buf; +} + +static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { + // taosMemoryFree(pSubTopicEp->schema.pSchema); + taosArrayDestroy(pSubTopicEp->vgs); +} + typedef struct { SMqRspHead head; int64_t reqOffset; @@ -2519,58 +2530,6 @@ typedef struct { SArray* topics; // SArray } SMqAskEpRsp; -static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { - // taosMemoryFree(pSubTopicEp->schema.pSchema); - taosArrayDestroy(pSubTopicEp->vgs); -} - -static FORCE_INLINE int32_t tEncodeSMqSubVgEp(void** buf, const SMqSubVgEp* pVgEp) { - int32_t tlen = 0; - tlen += taosEncodeFixedI32(buf, pVgEp->vgId); - tlen += taosEncodeFixedI64(buf, pVgEp->offset); - tlen += taosEncodeSEpSet(buf, &pVgEp->epSet); - return tlen; -} - -static FORCE_INLINE void* tDecodeSMqSubVgEp(void* buf, SMqSubVgEp* pVgEp) { - buf = taosDecodeFixedI32(buf, &pVgEp->vgId); - buf = taosDecodeFixedI64(buf, &pVgEp->offset); - buf = taosDecodeSEpSet(buf, &pVgEp->epSet); - return buf; -} - -static FORCE_INLINE int32_t tEncodeSMqSubTopicEp(void** buf, const SMqSubTopicEp* pTopicEp) { - int32_t tlen = 0; - tlen += taosEncodeString(buf, pTopicEp->topic); - tlen += taosEncodeFixedI8(buf, pTopicEp->isSchemaAdaptive); - int32_t sz = taosArrayGetSize(pTopicEp->vgs); - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { - SMqSubVgEp* pVgEp = (SMqSubVgEp*)taosArrayGet(pTopicEp->vgs, i); - tlen += tEncodeSMqSubVgEp(buf, pVgEp); - } - tlen += taosEncodeSSchemaWrapper(buf, &pTopicEp->schema); - return tlen; -} - -static FORCE_INLINE void* tDecodeSMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicEp) { - buf = taosDecodeStringTo(buf, pTopicEp->topic); - buf = taosDecodeFixedI8(buf, &pTopicEp->isSchemaAdaptive); - int32_t sz; - buf = taosDecodeFixedI32(buf, &sz); - pTopicEp->vgs = taosArrayInit(sz, sizeof(SMqSubVgEp)); - if (pTopicEp->vgs == NULL) { - return NULL; - } - for (int32_t i = 0; i < sz; i++) { - SMqSubVgEp vgEp; - buf = tDecodeSMqSubVgEp(buf, &vgEp); - taosArrayPush(pTopicEp->vgs, &vgEp); - } - buf = taosDecodeSSchemaWrapper(buf, &pTopicEp->schema); - return buf; -} - static FORCE_INLINE int32_t tEncodeSMqAskEpRsp(void** buf, const SMqAskEpRsp* pRsp) { int32_t tlen = 0; // tlen += taosEncodeString(buf, pRsp->cgroup); diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 93a950466b..e541c214de 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -208,6 +208,7 @@ int32_t walReadWithFp(SWal *, FWalWrite writeFp, int64_t verStart, int32_t readN int64_t walGetFirstVer(SWal *); int64_t walGetSnapshotVer(SWal *); int64_t walGetLastVer(SWal *); +int64_t walGetCommittedVer(SWal *); #ifdef __cplusplus } diff --git a/include/util/tencode.h b/include/util/tencode.h index 8504aec5ea..2a43d7934f 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -457,6 +457,55 @@ static FORCE_INLINE void* tDecoderMalloc(SDecoder* pCoder, int32_t size) { return p; } +static FORCE_INLINE int32_t tPutBinary(uint8_t* p, const uint8_t* pData, uint32_t nData) { + int n = 0; + uint32_t v = nData; + + for (;;) { + if (v <= 0x7f) { + if (p) p[n] = v; + n++; + break; + } + + if (p) p[n] = (v & 0x7f) | 0x80; + n++; + v >>= 7; + } + + if (p) { + memcpy(p + n, pData, nData); + } + n += nData; + + return n; +} + +static FORCE_INLINE int32_t tGetBinary(const uint8_t* p, const uint8_t** ppData, uint32_t* nData) { + int32_t n = 0; + uint32_t tv = 0; + uint32_t t; + + for (;;) { + if (p[n] <= 0x7f) { + t = p[n]; + tv |= (t << (7 * n)); + n++; + break; + } + + t = p[n] & 0x7f; + tv |= (t << (7 * n)); + n++; + } + + if (nData) *nData = n; + if (ppData) *ppData = p + n; + + n += tv; + return n; +} + #ifdef __cplusplus } #endif diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 3656edaff3..9909aa63f9 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -138,14 +138,14 @@ typedef struct { } SSmlHandle; //================================================================================================= -static uint64_t linesSmlHandleId = 0; +static volatile int64_t linesSmlHandleId = 0; static const char* TS = "_ts"; static const char* TAG = "_tagNone"; //================================================================================================= -static uint64_t smlGenId() { - uint64_t id; +static int64_t smlGenId() { + int64_t id; do { id = atomic_add_fetch_64(&linesSmlHandleId, 1); @@ -239,15 +239,13 @@ static int32_t smlApplySchemaAction(SSmlHandle* info, SSchemaAction* action) { TAOS_RES* res = taos_query(info->taos, result); //TODO async doAsyncQuery code = taos_errno(res); const char* errStr = taos_errstr(res); - char* begin = strstr(errStr, "duplicated column names"); - bool tscDupColNames = (begin != NULL); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%"PRIx64" apply schema action. error: %s", info->id, errStr); } taos_free_result(res); // if (code == TSDB_CODE_MND_FIELD_ALREADY_EXIST || code == TSDB_CODE_MND_TAG_ALREADY_EXIST || tscDupColNames) { - if (code == TSDB_CODE_MND_TAG_ALREADY_EXIST || tscDupColNames) { + if (code == TSDB_CODE_MND_TAG_ALREADY_EXIST) { TAOS_RES* res2 = taos_query(info->taos, "RESET QUERY CACHE"); code = taos_errno(res2); if (code != TSDB_CODE_SUCCESS) { @@ -265,15 +263,13 @@ static int32_t smlApplySchemaAction(SSmlHandle* info, SSchemaAction* action) { TAOS_RES* res = taos_query(info->taos, result); //TODO async doAsyncQuery code = taos_errno(res); const char* errStr = taos_errstr(res); - char* begin = strstr(errStr, "duplicated column names"); - bool tscDupColNames = (begin != NULL); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); } taos_free_result(res); // if (code ==TSDB_CODE_MND_TAG_ALREADY_EXIST || code == TSDB_CODE_MND_FIELD_ALREAY_EXIST || tscDupColNames) { - if (code ==TSDB_CODE_MND_TAG_ALREADY_EXIST || tscDupColNames) { + if (code ==TSDB_CODE_MND_TAG_ALREADY_EXIST) { TAOS_RES* res2 = taos_query(info->taos, "RESET QUERY CACHE"); code = taos_errno(res2); if (code != TSDB_CODE_SUCCESS) { @@ -337,7 +333,7 @@ static int32_t smlApplySchemaAction(SSmlHandle* info, SSchemaAction* action) { SArray *cols = action->createSTable.fields; for(int i = 0; i < taosArrayGetSize(cols); i++){ - SSmlKv *kv = taosArrayGetP(cols, i); + SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); smlBuildColumnDescription(kv, pos, freeBytes, &outBytes); pos += outBytes; freeBytes -= outBytes; *pos = ','; ++pos; --freeBytes; @@ -350,7 +346,7 @@ static int32_t smlApplySchemaAction(SSmlHandle* info, SSchemaAction* action) { cols = action->createSTable.tags; for(int i = 0; i < taosArrayGetSize(cols); i++){ - SSmlKv *kv = taosArrayGetP(cols, i); + SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); smlBuildColumnDescription(kv, pos, freeBytes, &outBytes); pos += outBytes; freeBytes -= outBytes; *pos = ','; ++pos; --freeBytes; @@ -390,7 +386,7 @@ static int32_t smlApplySchemaAction(SSmlHandle* info, SSchemaAction* action) { static int32_t smlModifyDBSchemas(SSmlHandle* info) { int32_t code = 0; - SSmlSTableMeta** tableMetaSml = taosHashIterate(info->superTables, NULL); + SSmlSTableMeta** tableMetaSml = (SSmlSTableMeta**)taosHashIterate(info->superTables, NULL); while (tableMetaSml) { SSmlSTableMeta* sTableData = *tableMetaSml; @@ -406,8 +402,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle* info) { code = catalogGetSTableMeta(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, &pTableMeta); if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_MND_INVALID_STB) { - SSchemaAction schemaAction = {0}; - schemaAction.action = SCHEMA_ACTION_CREATE_STABLE; + SSchemaAction schemaAction = { SCHEMA_ACTION_CREATE_STABLE, 0}; memcpy(schemaAction.createSTable.sTableName, superTable, superTableLen); schemaAction.createSTable.tags = sTableData->tags; schemaAction.createSTable.fields = sTableData->cols; @@ -430,7 +425,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle* info) { } sTableData->tableMeta = pTableMeta; - tableMetaSml = taosHashIterate(info->superTables, tableMetaSml); + tableMetaSml = (SSmlSTableMeta**)taosHashIterate(info->superTables, tableMetaSml); } return 0; } @@ -996,7 +991,7 @@ static int32_t smlParseString(const char* sql, SSmlLineInfo *elements, SSmlMsgBu static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, bool isTag, SHashObj *dumplicateKey, SSmlMsgBuf *msg){ if(isTag && len == 0){ - SSmlKv *kv = taosMemoryCalloc(sizeof(SSmlKv), 1); + SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); kv->key = TAG; kv->keyLen = strlen(TAG); kv->value = TAG; @@ -1053,7 +1048,7 @@ static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, bool is } // add kv to SSmlKv - SSmlKv *kv = taosMemoryCalloc(sizeof(SSmlKv), 1); + SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); kv->key = key; kv->keyLen = keyLen; kv->value = value; @@ -1199,7 +1194,7 @@ static int32_t smlParseTS(SSmlHandle* info, const char* data, int32_t len, SArra if(ts == -1) return TSDB_CODE_TSC_INVALID_TIME_STAMP; // add ts to - SSmlKv *kv = taosMemoryCalloc(sizeof(SSmlKv), 1); + SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); if(!kv){ return TSDB_CODE_OUT_OF_MEMORY; } @@ -1259,12 +1254,12 @@ static int32_t smlParseTS(SSmlHandle* info, const char* data, int32_t len, SArra static bool smlUpdateMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols, SSmlMsgBuf *msg){ if(tags){ for (int i = 0; i < taosArrayGetSize(tags); ++i) { - SSmlKv *kv = taosArrayGetP(tags, i); + SSmlKv *kv = (SSmlKv *)taosArrayGetP(tags, i); ASSERT(kv->type == TSDB_DATA_TYPE_NCHAR); - uint8_t *index = taosHashGet(tableMeta->tagHash, kv->key, kv->keyLen); + uint8_t *index = (uint8_t *)taosHashGet(tableMeta->tagHash, kv->key, kv->keyLen); if(index){ - SSmlKv **value = taosArrayGet(tableMeta->tags, *index); + SSmlKv **value = (SSmlKv **)taosArrayGet(tableMeta->tags, *index); ASSERT((*value)->type == TSDB_DATA_TYPE_NCHAR); if(kv->valueLen > (*value)->valueLen){ // tags type is nchar *value = kv; @@ -1281,11 +1276,11 @@ static bool smlUpdateMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols, if(cols){ for (int i = 1; i < taosArrayGetSize(cols); ++i) { //jump timestamp - SSmlKv *kv = taosArrayGetP(cols, i); + SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); - int16_t *index = taosHashGet(tableMeta->fieldHash, kv->key, kv->keyLen); + int16_t *index = (int16_t *)taosHashGet(tableMeta->fieldHash, kv->key, kv->keyLen); if(index){ - SSmlKv **value = taosArrayGet(tableMeta->cols, *index); + SSmlKv **value = (SSmlKv **)taosArrayGet(tableMeta->cols, *index); if(kv->type != (*value)->type){ smlBuildInvalidDataMsg(msg, "the type is not the same like before", kv->key); return false; @@ -1311,7 +1306,7 @@ static bool smlUpdateMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols, static void smlInsertMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols){ if(tags){ for (uint8_t i = 0; i < taosArrayGetSize(tags); ++i) { - SSmlKv *kv = taosArrayGetP(tags, i); + SSmlKv *kv = (SSmlKv *)taosArrayGetP(tags, i); taosArrayPush(tableMeta->tags, &kv); taosHashPut(tableMeta->tagHash, kv->key, kv->keyLen, &i, CHAR_BYTES); } @@ -1319,7 +1314,7 @@ static void smlInsertMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols) if(cols){ for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) { - SSmlKv *kv = taosArrayGetP(cols, i); + SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); taosArrayPush(tableMeta->cols, &kv); taosHashPut(tableMeta->fieldHash, kv->key, kv->keyLen, &i, SHORT_BYTES); } @@ -1327,7 +1322,7 @@ static void smlInsertMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols) } static SSmlTableInfo* smlBuildTableInfo(bool format){ - SSmlTableInfo *tag = taosMemoryCalloc(sizeof(SSmlTableInfo), 1); + SSmlTableInfo *tag = (SSmlTableInfo *)taosMemoryCalloc(sizeof(SSmlTableInfo), 1); if(!tag){ return NULL; } @@ -1354,7 +1349,7 @@ static SSmlTableInfo* smlBuildTableInfo(bool format){ return tag; cleanup: - taosMemoryFreeClear(tag); + taosMemoryFree(tag); return NULL; } @@ -1364,18 +1359,17 @@ static void smlDestroyBuildTableInfo(SSmlTableInfo *tag, bool format){ }else{ tag->cols = taosArrayInit(16, POINTER_BYTES); for(size_t i = 0; i < taosArrayGetSize(tag->cols); i++){ - SHashObj *kvHash = taosArrayGetP(tag->cols, i); - void** p1 = taosHashIterate(kvHash, NULL); + SHashObj *kvHash = (SHashObj *)taosArrayGetP(tag->cols, i); + void** p1 = (void**)taosHashIterate(kvHash, NULL); while (p1) { - SSmlKv* kv = *p1; - taosMemoryFreeClear(kv); - p1 = taosHashIterate(kvHash, p1); + taosMemoryFree(*p1); + p1 = (void**)taosHashIterate(kvHash, p1); } taosHashCleanup(kvHash); } } taosArrayDestroy(tag->tags); - taosMemoryFreeClear(tag); + taosMemoryFree(tag); } static int32_t smlDealCols(SSmlTableInfo* oneTable, bool dataFormat, SArray *cols){ @@ -1390,7 +1384,7 @@ static int32_t smlDealCols(SSmlTableInfo* oneTable, bool dataFormat, SArray *col return TSDB_CODE_TSC_OUT_OF_MEMORY; } for(size_t i = 0; i < taosArrayGetSize(cols); i++){ - SSmlKv *kv = taosArrayGetP(cols, i); + SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); // todo key need escape, like \=, because find by schema name later } taosArrayPush(oneTable->cols, &kvHash); @@ -1399,7 +1393,7 @@ static int32_t smlDealCols(SSmlTableInfo* oneTable, bool dataFormat, SArray *col } static SSmlSTableMeta* smlBuildSTableMeta(){ - SSmlSTableMeta* meta = taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); + SSmlSTableMeta* meta = (SSmlSTableMeta*)taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); if(!meta){ return NULL; } @@ -1429,7 +1423,7 @@ static SSmlSTableMeta* smlBuildSTableMeta(){ return meta; cleanup: - taosMemoryFreeClear(meta); + taosMemoryFree(meta); return NULL; } @@ -1475,9 +1469,9 @@ static int32_t smlParseLine(SSmlHandle* info, const char* sql) { return TSDB_CODE_SML_INVALID_DATA; } - SSmlTableInfo **oneTable = taosHashGet(info->childTables, elements.measure, elements.measureTagsLen); + SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashGet(info->childTables, elements.measure, elements.measureTagsLen); if(oneTable){ - SSmlSTableMeta** tableMeta = taosHashGet(info->superTables, elements.measure, elements.measureLen); + SSmlSTableMeta** tableMeta = (SSmlSTableMeta**)taosHashGet(info->superTables, elements.measure, elements.measureLen); ASSERT(tableMeta); ret = smlUpdateMeta(*tableMeta, NULL, cols, &info->msgBuf); // update meta cols if(!ret){ @@ -1511,12 +1505,12 @@ static int32_t smlParseLine(SSmlHandle* info, const char* sql) { tinfo->sTableName = elements.measure; tinfo->sTableNameLen = elements.measureLen; - RandTableName rName = {.tags=tinfo->tags, .sTableName=tinfo->sTableName, .sTableNameLen=tinfo->sTableNameLen, - .childTableName=tinfo->childTableName}; + RandTableName rName = { tinfo->tags, tinfo->sTableName, tinfo->sTableNameLen, + tinfo->childTableName, 0 }; buildChildTableName(&rName); tinfo->uid = rName.uid; - SSmlSTableMeta** tableMeta = taosHashGet(info->superTables, elements.measure, elements.measureLen); + SSmlSTableMeta** tableMeta = (SSmlSTableMeta**)taosHashGet(info->superTables, elements.measure, elements.measureLen); if(tableMeta){ // update meta ret = smlUpdateMeta(*tableMeta, tinfo->tags, cols, &info->msgBuf); if(!ret){ @@ -1545,20 +1539,18 @@ static void smlDestroyInfo(SSmlHandle* info){ smlDestroyHandle(info->exec); // destroy info->childTables - void** p1 = taosHashIterate(info->childTables, NULL); + void** p1 = (void**)taosHashIterate(info->childTables, NULL); while (p1) { - SSmlTableInfo* oneTable = *p1; - smlDestroyBuildTableInfo(oneTable, info->dataFormat); - p1 = taosHashIterate(info->childTables, p1); + smlDestroyBuildTableInfo((SSmlTableInfo*)(*p1), info->dataFormat); + p1 = (void**)taosHashIterate(info->childTables, p1); } taosHashCleanup(info->childTables); // destroy info->superTables - p1 = taosHashIterate(info->superTables, NULL); + p1 = (void**)taosHashIterate(info->superTables, NULL); while (p1) { - SSmlSTableMeta* oneTable = *p1; - smlDestroySTableMeta(oneTable); - p1 = taosHashIterate(info->superTables, p1); + smlDestroySTableMeta((SSmlSTableMeta*)(*p1)); + p1 = (void**)taosHashIterate(info->superTables, p1); } taosHashCleanup(info->superTables); @@ -1571,13 +1563,13 @@ static void smlDestroyInfo(SSmlHandle* info){ static SSmlHandle* smlBuildSmlInfo(TAOS* taos, SRequestObj* request, SMLProtocolType protocol, int8_t precision, bool dataFormat){ int32_t code = TSDB_CODE_SUCCESS; - SSmlHandle* info = taosMemoryCalloc(1, sizeof(SSmlHandle)); + SSmlHandle* info = (SSmlHandle*)taosMemoryCalloc(1, sizeof(SSmlHandle)); if (NULL == info) { return NULL; } info->id = smlGenId(); - info->pQuery = taosMemoryCalloc(1, sizeof(SQuery)); + info->pQuery = (SQuery *)taosMemoryCalloc(1, sizeof(SQuery)); if (NULL == info->pQuery) { uError("SML:0x%"PRIx64" create info->pQuery error", info->id); goto cleanup; @@ -1592,7 +1584,7 @@ static SSmlHandle* smlBuildSmlInfo(TAOS* taos, SRequestObj* request, SMLProtocol } ((SVnodeModifOpStmt*)(info->pQuery->pRoot))->payloadType = PAYLOAD_TYPE_KV; - info->taos = taos; + info->taos = (STscObj *)taos; code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); if(code != TSDB_CODE_SUCCESS){ uError("SML:0x%"PRIx64" get catalog error %d", info->id, code); @@ -1634,7 +1626,7 @@ cleanup: static int32_t smlInsertData(SSmlHandle* info) { int32_t code = TSDB_CODE_SUCCESS; - SSmlTableInfo** oneTable = taosHashIterate(info->childTables, NULL); + SSmlTableInfo** oneTable = (SSmlTableInfo**)taosHashIterate(info->childTables, NULL); while (oneTable) { SSmlTableInfo* tableData = *oneTable; @@ -1650,7 +1642,7 @@ static int32_t smlInsertData(SSmlHandle* info) { } taosHashPut(info->pVgHash, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)); - SSmlSTableMeta** pMeta = taosHashGet(info->superTables, tableData->sTableName, tableData->sTableNameLen); + SSmlSTableMeta** pMeta = (SSmlSTableMeta** )taosHashGet(info->superTables, tableData->sTableName, tableData->sTableNameLen); ASSERT (NULL != pMeta && NULL != *pMeta); // use tablemeta of stable to save vgid and uid of child table @@ -1662,7 +1654,7 @@ static int32_t smlInsertData(SSmlHandle* info) { if(code != TSDB_CODE_SUCCESS){ return code; } - oneTable = taosHashIterate(info->childTables, oneTable); + oneTable = (SSmlTableInfo**)taosHashIterate(info->childTables, oneTable); } smlBuildOutput(info->exec, info->pVgHash); @@ -1748,12 +1740,12 @@ cleanup: */ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision) { - SRequestObj* request = createRequest(taos, NULL, NULL, TSDB_SQL_INSERT); + SRequestObj* request = (SRequestObj*)createRequest((STscObj *)taos, NULL, NULL, TSDB_SQL_INSERT); if(!request){ return NULL; } - SSmlHandle* info = smlBuildSmlInfo(taos, request, protocol, precision, true); + SSmlHandle* info = smlBuildSmlInfo(taos, request, (SMLProtocolType)protocol, precision, true); if(!info){ return (TAOS_RES*)request; } diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 6a4823b855..89a573e4ee 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -214,7 +214,7 @@ TEST(testCase, smlParseCols_tag_Test) { msgBuf.len = 256; SArray *cols = taosArrayInit(16, POINTER_BYTES); - ASSERT_NE(cols, NULL); + ASSERT_NE(cols, nullptr); SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); const char *data = @@ -226,7 +226,7 @@ TEST(testCase, smlParseCols_tag_Test) { ASSERT_EQ(size, 19); // nchar - SSmlKv *kv = taosArrayGetP(cols, 0); + SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, 0); ASSERT_EQ(strncasecmp(kv->key, "cbin", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); @@ -235,7 +235,7 @@ TEST(testCase, smlParseCols_tag_Test) { taosMemoryFree(kv); // nchar - kv = taosArrayGetP(cols, 3); + kv = (SSmlKv *)taosArrayGetP(cols, 3); ASSERT_EQ(strncasecmp(kv->key, "cf64", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); @@ -257,7 +257,7 @@ TEST(testCase, smlParseCols_tag_Test) { ASSERT_EQ(size, 1); // nchar - kv = taosArrayGetP(cols, 0); + kv = (SSmlKv *)taosArrayGetP(cols, 0); ASSERT_EQ(strncasecmp(kv->key, TAG, strlen(TAG)), 0); ASSERT_EQ(kv->keyLen, strlen(TAG)); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); @@ -276,7 +276,7 @@ TEST(testCase, smlParseCols_Test) { msgBuf.len = 256; SArray *cols = taosArrayInit(16, POINTER_BYTES); - ASSERT_NE(cols, NULL); + ASSERT_NE(cols, nullptr); SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); @@ -288,7 +288,7 @@ TEST(testCase, smlParseCols_Test) { ASSERT_EQ(size, 19); // binary - SSmlKv *kv = taosArrayGetP(cols, 0); + SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, 0); ASSERT_EQ(strncasecmp(kv->key, "cbin", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BINARY); @@ -297,7 +297,7 @@ TEST(testCase, smlParseCols_Test) { taosMemoryFree(kv); // nchar - kv = taosArrayGetP(cols, 1); + kv = (SSmlKv *)taosArrayGetP(cols, 1); ASSERT_EQ(strncasecmp(kv->key, "cnch", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); @@ -306,7 +306,7 @@ TEST(testCase, smlParseCols_Test) { taosMemoryFree(kv); // bool - kv = taosArrayGetP(cols, 2); + kv = (SSmlKv *)taosArrayGetP(cols, 2); ASSERT_EQ(strncasecmp(kv->key, "cbool", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); @@ -315,7 +315,7 @@ TEST(testCase, smlParseCols_Test) { taosMemoryFree(kv); // double - kv = taosArrayGetP(cols, 3); + kv = (SSmlKv *)taosArrayGetP(cols, 3); ASSERT_EQ(strncasecmp(kv->key, "cf64", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE); @@ -325,7 +325,7 @@ TEST(testCase, smlParseCols_Test) { taosMemoryFree(kv); // float - kv = taosArrayGetP(cols, 4); + kv = (SSmlKv *)taosArrayGetP(cols, 4); ASSERT_EQ(strncasecmp(kv->key, "cf32_", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_FLOAT); @@ -335,7 +335,7 @@ TEST(testCase, smlParseCols_Test) { taosMemoryFree(kv); // float - kv = taosArrayGetP(cols, 5); + kv = (SSmlKv *)taosArrayGetP(cols, 5); ASSERT_EQ(strncasecmp(kv->key, "cf32", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_FLOAT); @@ -345,7 +345,7 @@ TEST(testCase, smlParseCols_Test) { taosMemoryFree(kv); // tiny int - kv = taosArrayGetP(cols, 6); + kv = (SSmlKv *)taosArrayGetP(cols, 6); ASSERT_EQ(strncasecmp(kv->key, "ci8", 3), 0); ASSERT_EQ(kv->keyLen, 3); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_TINYINT); @@ -354,7 +354,7 @@ TEST(testCase, smlParseCols_Test) { taosMemoryFree(kv); // unsigned tiny int - kv = taosArrayGetP(cols, 7); + kv = (SSmlKv *)taosArrayGetP(cols, 7); ASSERT_EQ(strncasecmp(kv->key, "cu8", 3), 0); ASSERT_EQ(kv->keyLen, 3); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UTINYINT); @@ -363,7 +363,7 @@ TEST(testCase, smlParseCols_Test) { taosMemoryFree(kv); // small int - kv = taosArrayGetP(cols, 8); + kv = (SSmlKv *)taosArrayGetP(cols, 8); ASSERT_EQ(strncasecmp(kv->key, "ci16", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_SMALLINT); @@ -372,7 +372,7 @@ TEST(testCase, smlParseCols_Test) { taosMemoryFree(kv); // unsigned smallint - kv = taosArrayGetP(cols, 9); + kv = (SSmlKv *)taosArrayGetP(cols, 9); ASSERT_EQ(strncasecmp(kv->key, "cu16", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_USMALLINT); @@ -381,7 +381,7 @@ TEST(testCase, smlParseCols_Test) { taosMemoryFree(kv); // int - kv = taosArrayGetP(cols, 10); + kv = (SSmlKv *)taosArrayGetP(cols, 10); ASSERT_EQ(strncasecmp(kv->key, "ci32", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_INT); @@ -390,7 +390,7 @@ TEST(testCase, smlParseCols_Test) { taosMemoryFree(kv); // unsigned int - kv = taosArrayGetP(cols, 11); + kv = (SSmlKv *)taosArrayGetP(cols, 11); ASSERT_EQ(strncasecmp(kv->key, "cu32", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UINT); @@ -400,7 +400,7 @@ TEST(testCase, smlParseCols_Test) { // bigint - kv = taosArrayGetP(cols, 12); + kv = (SSmlKv *)taosArrayGetP(cols, 12); ASSERT_EQ(strncasecmp(kv->key, "ci64", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT); @@ -409,7 +409,7 @@ TEST(testCase, smlParseCols_Test) { taosMemoryFree(kv); // bigint - kv = taosArrayGetP(cols, 13); + kv = (SSmlKv *)taosArrayGetP(cols, 13); ASSERT_EQ(strncasecmp(kv->key, "ci", 2), 0); ASSERT_EQ(kv->keyLen, 2); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT); @@ -418,7 +418,7 @@ TEST(testCase, smlParseCols_Test) { taosMemoryFree(kv); // unsigned bigint - kv = taosArrayGetP(cols, 14); + kv = (SSmlKv *)taosArrayGetP(cols, 14); ASSERT_EQ(strncasecmp(kv->key, "cu64", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UBIGINT); @@ -427,7 +427,7 @@ TEST(testCase, smlParseCols_Test) { taosMemoryFree(kv); // bool - kv = taosArrayGetP(cols, 15); + kv = (SSmlKv *)taosArrayGetP(cols, 15); ASSERT_EQ(strncasecmp(kv->key, "cbooltrue", 9), 0); ASSERT_EQ(kv->keyLen, 9); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); @@ -437,7 +437,7 @@ TEST(testCase, smlParseCols_Test) { // bool - kv = taosArrayGetP(cols, 16); + kv = (SSmlKv *)taosArrayGetP(cols, 16); ASSERT_EQ(strncasecmp(kv->key, "cboolt", 6), 0); ASSERT_EQ(kv->keyLen, 6); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); @@ -446,7 +446,7 @@ TEST(testCase, smlParseCols_Test) { taosMemoryFree(kv); // bool - kv = taosArrayGetP(cols, 17); + kv = (SSmlKv *)taosArrayGetP(cols, 17); ASSERT_EQ(strncasecmp(kv->key, "cboolf", 6), 0); ASSERT_EQ(kv->keyLen, 6); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); @@ -455,7 +455,7 @@ TEST(testCase, smlParseCols_Test) { taosMemoryFree(kv); // nchar - kv = taosArrayGetP(cols, 18); + kv = (SSmlKv *)taosArrayGetP(cols, 18); ASSERT_EQ(strncasecmp(kv->key, "cnch_", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); @@ -469,7 +469,7 @@ TEST(testCase, smlParseCols_Test) { TEST(testCase, smlParseLine_Test) { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - ASSERT_NE(taos, NULL); + ASSERT_NE(taos, nullptr); TAOS_RES* pRes = taos_query(taos, "create database if not exists sml_db"); taos_free_result(pRes); @@ -477,11 +477,11 @@ TEST(testCase, smlParseLine_Test) { pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - SRequestObj *request = createRequest(taos, NULL, NULL, TSDB_SQL_INSERT); - ASSERT_NE(request, NULL); + SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT); + ASSERT_NE(request, nullptr); SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true); - ASSERT_NE(info, NULL); + ASSERT_NE(info, nullptr); const char *sql[9] = { "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 load_capacity=1500,fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=124,velocity=0,heading=221,grade=0 1451606400000000000", @@ -494,7 +494,7 @@ TEST(testCase, smlParseLine_Test) { "readings,name=truck_2,fleet=North,driver=Derek,model=F-150 load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=24.5208,longitude=28.09377,elevation=428,velocity=0,heading=304,grade=0,fuel_consumption=25 1451609400000000000", "readings,fleet=South,name=truck_0,driver=Trish,model=H-2,device_version=v2.3 fuel_consumption=25,grade=0 1451629400000000000" }; - smlInsertLines(info, sql, 9); + smlInsertLines(info, (char**)sql, 9); // for (int i = 0; i < 3; i++) { // smlParseLine(info, sql[i]); // } @@ -502,7 +502,7 @@ TEST(testCase, smlParseLine_Test) { TEST(testCase, smlParseLine_error_Test) { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - ASSERT_NE(taos, NULL); + ASSERT_NE(taos, nullptr); TAOS_RES* pRes = taos_query(taos, "create database if not exists sml_db"); taos_free_result(pRes); @@ -510,17 +510,17 @@ TEST(testCase, smlParseLine_error_Test) { pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - SRequestObj *request = createRequest(taos, NULL, NULL, TSDB_SQL_INSERT); - ASSERT_NE(request, NULL); + SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT); + ASSERT_NE(request, nullptr); SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true); - ASSERT_NE(info, NULL); + ASSERT_NE(info, nullptr); const char *sql[2] = { "measure,t1=3 c1=8", "measure,t2=3 c1=8u8" }; - int ret = smlInsertLines(info, sql, 2); + int ret = smlInsertLines(info, (char **)sql, 2); ASSERT_NE(ret, 0); } diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 5d893fe398..ad020fe7d9 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -19,12 +19,238 @@ #include "tdatablock.h" #include "tlog.h" +#define TD_KV_ROW 0x1U + +struct SKVIdx { + int32_t cid; + int32_t offset; +}; + +int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow) { + if (tEncodeI64(pEncoder, pRow->ts) < 0) return -1; + if (tEncodeU32v(pEncoder, pRow->flags) < 0) return -1; + if (pRow->flags & TD_KV_ROW) { + if (tEncodeI32v(pEncoder, pRow->ncols) < 0) return -1; + } else { + if (tEncodeI32v(pEncoder, pRow->sver) < 0) return -1; + } + if (tEncodeBinary(pEncoder, pRow->pData, pRow->nData) < 0) return -1; + return 0; +} + +int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow) { + if (tDecodeI64(pDecoder, &pRow->ts) < 0) return -1; + if (tDecodeU32v(pDecoder, &pRow->flags) < 0) return -1; + if (pRow->flags & TD_KV_ROW) { + if (tDecodeI32v(pDecoder, &pRow->ncols) < 0) return -1; + } else { + if (tDecodeI32v(pDecoder, &pRow->sver) < 0) return -1; + } + if (tDecodeBinary(pDecoder, &pRow->pData, &pRow->nData) < 0) return -1; + return 0; +} + +int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t ncols, STSchema **ppTSchema) { + *ppTSchema = (STSchema *)taosMemoryMalloc(sizeof(STSchema) + sizeof(STColumn) * ncols); + if (*ppTSchema == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + (*ppTSchema)->numOfCols = ncols; + (*ppTSchema)->version = sver; + (*ppTSchema)->flen = 0; + (*ppTSchema)->vlen = 0; + (*ppTSchema)->tlen = 0; + + for (int32_t iCol = 0; iCol < ncols; iCol++) { + SSchema *pColumn = &pSchema[iCol]; + STColumn *pTColumn = &((*ppTSchema)->columns[iCol]); + + pTColumn->colId = pColumn->colId; + pTColumn->type = pColumn->type; + pTColumn->flags = pColumn->flags; + pTColumn->bytes = pColumn->bytes; + pTColumn->offset = (*ppTSchema)->flen; + + // skip first column + if (iCol) { + (*ppTSchema)->flen += TYPE_BYTES[pColumn->type]; + if (IS_VAR_DATA_TYPE(pColumn->type)) { + (*ppTSchema)->vlen += (pColumn->bytes + 5); + } + } + } + + return 0; +} + +void tTSchemaDestroy(STSchema *pTSchema) { taosMemoryFree(pTSchema); } + +int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchema, int32_t nCols) { + int32_t kvBufLen; + int32_t tpBufLen; + uint8_t *p; + + if (tTSchemaCreate(sver, pSchema, nCols, &pBuilder->pTSchema) < 0) return -1; + + kvBufLen = sizeof(SKVIdx) * nCols + pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen; + tpBufLen = pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen; + + if (pBuilder->szKVBuf < kvBufLen) { + p = taosMemoryRealloc(pBuilder->pKVBuf, kvBufLen); + if (p == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pBuilder->pKVBuf = p; + pBuilder->szKVBuf = kvBufLen; + } + + if (pBuilder->szTPBuf < tpBufLen) { + p = taosMemoryRealloc(pBuilder->pTPBuf, tpBufLen); + if (p == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pBuilder->pTPBuf = p; + pBuilder->szTPBuf = tpBufLen; + } + + tTSRowBuilderReset(pBuilder); + + return 0; +} + +void tTSRowBuilderClear(STSRowBuilder *pBuilder) { + taosMemoryFree(pBuilder->pKVBuf); + taosMemoryFree(pBuilder->pTPBuf); +} + +void tTSRowBuilderReset(STSRowBuilder *pBuilder) { + for (int32_t iCol = pBuilder->pTSchema->numOfCols - 1; iCol >= 0; iCol--) { + pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol]; + + pBuilder->pTColumn->flags &= (~COL_VAL_SET); + } + + pBuilder->nCols = 0; + pBuilder->kvVLen = 0; + pBuilder->tpVLen = 0; + pBuilder->row.flags = 0; +} + +int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pData, uint32_t nData) { + int32_t iCol; + uint8_t *p; + + // search column + if (pBuilder->pTColumn->colId < cid) { + iCol = (pBuilder->pTColumn - pBuilder->pTSchema->columns) / sizeof(STColumn) + 1; + for (; iCol < pBuilder->pTSchema->numOfCols; iCol++) { + pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol]; + if (pBuilder->pTColumn->colId == cid) break; + } + } else if (pBuilder->pTColumn->colId > cid) { + iCol = (pBuilder->pTColumn - pBuilder->pTSchema->columns) / sizeof(STColumn) - 1; + for (; iCol >= 0; iCol--) { + pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol]; + if (pBuilder->pTColumn->colId == cid) break; + } + } + + // check + if (pBuilder->pTColumn->colId != cid || pBuilder->pTColumn->flags & COL_VAL_SET) { + return -1; + } + + // set value + if (cid == 0) { + ASSERT(pData && nData == sizeof(TSKEY)); + pBuilder->row.ts = *(TSKEY *)pData; + } else { + if (pData) { + // ASSERT(!IS_NULL(pData)); + + // set tuple data + p = pBuilder->pTPBuf + pBuilder->pTColumn->offset; + if (IS_VAR_DATA_TYPE(pBuilder->pTColumn->type)) { + *(int32_t *)p = pBuilder->tpVLen; + + // encode the variant-length data + p = pBuilder->pTPBuf + pBuilder->pTSchema->flen + pBuilder->tpVLen; + pBuilder->tpVLen += tPutBinary(p, pData, nData); + } else { + memcpy(p, pData, nData); + } + + // set kv data + p = pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols; + ((SKVIdx *)p)->cid = cid; + ((SKVIdx *)p)->offset = pBuilder->kvVLen; + + p = pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->pTSchema->numOfCols + pBuilder->kvVLen; + if (IS_VAR_DATA_TYPE(pBuilder->pTColumn->type)) { + pBuilder->kvVLen += tPutBinary(p, pData, nData); + } else { + memcpy(p, pData, nData); + pBuilder->kvVLen += nData; + } + } else { + // set NULL val + } + } + + pBuilder->pTColumn->flags |= COL_VAL_SET; + pBuilder->nCols++; + return 0; +} + +int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) { + if ((pBuilder->pTSchema->columns[0].flags & COL_VAL_SET) == 0) { + return -1; + } + + if (pBuilder->nCols * sizeof(SKVIdx) + pBuilder->kvVLen < pBuilder->pTSchema->flen + pBuilder->tpVLen) { + // encode as TD_KV_ROW + pBuilder->row.flags |= TD_KV_ROW; + pBuilder->row.ncols = pBuilder->nCols; + pBuilder->row.nData = pBuilder->nCols * sizeof(SKVIdx) + pBuilder->kvVLen; + pBuilder->row.pData = pBuilder->pKVBuf; + + if (pBuilder->nCols < pBuilder->pTSchema->numOfCols) { + memmove(pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols, + pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->pTSchema->numOfCols, pBuilder->kvVLen); + } + } else { + // encode as TD_TUPLE_ROW + pBuilder->row.flags &= (~TD_KV_ROW); + pBuilder->row.sver = pBuilder->pTSchema->version; + pBuilder->row.nData = pBuilder->pTSchema->flen + pBuilder->tpVLen; + pBuilder->row.pData = pBuilder->pTPBuf; + + if (pBuilder->nCols < pBuilder->pTSchema->numOfCols) { + // set non-set cols as None + for (int32_t iCol = 1; iCol < pBuilder->pTSchema->numOfCols; iCol++) { + pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol]; + if (pBuilder->pTColumn->flags & COL_VAL_SET) continue; + + { + // set None (todo) + } + + pBuilder->pTColumn->flags |= COL_VAL_SET; + } + } + } + + *ppRow = &pBuilder->row; + return 0; +} + +#if 1 // ==================== static void dataColSetNEleNull(SDataCol *pCol, int nEle); -#if 0 -static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, - int limit2, int tRows, bool forceSetNull); -#endif -int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { +int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { int spaceNeeded = pCol->bytes * maxPoints; if (IS_VAR_DATA_TYPE(pCol->type)) { spaceNeeded += sizeof(VarDataOffsetT) * maxPoints; @@ -504,3 +730,4 @@ SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder) { return row; } +#endif \ No newline at end of file diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index f017d22b39..f73a982110 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -351,7 +351,7 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) { } static int32_t taosAddServerCfg(SConfig *pCfg) { - if (cfgAddInt32(pCfg, "supportVnodes", 256, 0, 65536, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "supportVnodes", 256, 0, 4096, 0) != 0) return -1; if (cfgAddDir(pCfg, "dataDir", tsDataDir, 0) != 0) return -1; if (cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "maxNumOfDistinctRes", tsMaxNumOfDistinctResults, 10 * 10000, 10000 * 10000, 0) != 0) return -1; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index c5919e06b6..6a0f4984b1 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -140,11 +140,17 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->vgId = pCreate->vgId; tstrncpy(pCfg->dbname, pCreate->db, sizeof(pCfg->dbname)); pCfg->dbId = pCreate->dbUid; + pCfg->szPage = pCreate->pageSize * 1024; + pCfg->szCache = pCreate->pages; + pCfg->szBuf = pCreate->buffer * 1024 * 1024; pCfg->isWeak = true; + pCfg->tsdbCfg.precision = pCreate->precision; pCfg->tsdbCfg.days = 10; pCfg->tsdbCfg.keep0 = 3650; pCfg->tsdbCfg.keep1 = 3650; pCfg->tsdbCfg.keep2 = 3650; + pCfg->tsdbCfg.minRows = pCreate->minRows; + pCfg->tsdbCfg.maxRows = pCreate->maxRows; for (size_t i = 0; i < taosArrayGetSize(pCreate->pRetensions); ++i) { memcpy(&pCfg->tsdbCfg.retentions[i], taosArrayGet(pCreate->pRetensions, i), sizeof(SRetention)); } diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 2751e0752e..5eb6866387 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -452,6 +452,7 @@ typedef struct { int8_t withSchema; int8_t withTag; SRWLatch lock; + int32_t consumerCnt; int32_t sqlLen; int32_t astLen; char* sql; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 645f2ff0e7..c84cc10050 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -412,8 +412,8 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->numOfVgroups < 0) pCfg->numOfVgroups = TSDB_DEFAULT_VN_PER_DB; if (pCfg->numOfStables < 0) pCfg->numOfStables = TSDB_DEFAULT_DB_SINGLE_STABLE; if (pCfg->buffer < 0) pCfg->buffer = TSDB_DEFAULT_BUFFER_PER_VNODE; - if (pCfg->pageSize < 0) pCfg->pageSize = TSDB_DEFAULT_PAGES_PER_VNODE; - if (pCfg->pages < 0) pCfg->pages = TSDB_MAX_PAGESIZE_PER_VNODE; + if (pCfg->pageSize < 0) pCfg->pageSize = TSDB_DEFAULT_PAGESIZE_PER_VNODE; + if (pCfg->pages < 0) pCfg->pages = TSDB_DEFAULT_PAGES_PER_VNODE; if (pCfg->daysPerFile < 0) pCfg->daysPerFile = TSDB_DEFAULT_DURATION_PER_FILE; if (pCfg->daysToKeep0 < 0) pCfg->daysToKeep0 = TSDB_DEFAULT_KEEP; if (pCfg->daysToKeep1 < 0) pCfg->daysToKeep1 = pCfg->daysToKeep0; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 179f4ba24c..ca0ae111a3 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -832,6 +832,8 @@ static int32_t mndProcessVCreateStbRsp(SNodeMsg *pRsp) { } static int32_t mndCheckAlterStbReq(SMAlterStbReq *pAlter) { + if (pAlter->commentLen != 0) return 0; + if (pAlter->numOfFields < 1 || pAlter->numOfFields != (int32_t)taosArrayGetSize(pAlter->pFields)) { terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; @@ -839,15 +841,6 @@ static int32_t mndCheckAlterStbReq(SMAlterStbReq *pAlter) { for (int32_t i = 0; i < pAlter->numOfFields; ++i) { SField *pField = taosArrayGet(pAlter->pFields, i); - - if (pField->type <= 0) { - terrno = TSDB_CODE_MND_INVALID_STB_OPTION; - return -1; - } - if (pField->bytes <= 0) { - terrno = TSDB_CODE_MND_INVALID_STB_OPTION; - return -1; - } if (pField->name[0] == 0) { terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; @@ -890,6 +883,23 @@ static int32_t mndAllocStbSchemas(const SStbObj *pOld, SStbObj *pNew) { return 0; } +static int32_t mndUpdateStbComment(const SStbObj *pOld, SStbObj *pNew, char *pComment, int32_t commentLen) { + if (commentLen > 0) { + pNew->commentLen = commentLen; + pNew->comment = taosMemoryCalloc(1, commentLen); + if (pNew->comment == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + memcpy(pNew->comment, pComment, commentLen); + } + + if (mndAllocStbSchemas(pOld, pNew) != 0) { + return -1; + } + return 0; +} + static int32_t mndAddSuperTableTag(const SStbObj *pOld, SStbObj *pNew, SArray *pFields, int32_t ntags) { if (pOld->numOfTags + ntags > TSDB_MAX_TAGS) { terrno = TSDB_CODE_MND_TOO_MANY_TAGS; @@ -908,12 +918,12 @@ static int32_t mndAddSuperTableTag(const SStbObj *pOld, SStbObj *pNew, SArray *p for (int32_t i = 0; i < ntags; i++) { SField *pField = taosArrayGet(pFields, i); - if (mndFindSuperTableColumnIndex(pOld, pField->name) > 0) { + if (mndFindSuperTableColumnIndex(pOld, pField->name) >= 0) { terrno = TSDB_CODE_MND_COLUMN_ALREADY_EXIST; return -1; } - if (mndFindSuperTableTagIndex(pOld, pField->name) > 0) { + if (mndFindSuperTableTagIndex(pOld, pField->name) >= 0) { terrno = TSDB_CODE_MND_TAG_ALREADY_EXIST; return -1; } @@ -1034,12 +1044,12 @@ static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray for (int32_t i = 0; i < ncols; i++) { SField *pField = taosArrayGet(pFields, i); - if (mndFindSuperTableColumnIndex(pOld, pField->name) > 0) { + if (mndFindSuperTableColumnIndex(pOld, pField->name) >= 0) { terrno = TSDB_CODE_MND_COLUMN_ALREADY_EXIST; return -1; } - if (mndFindSuperTableTagIndex(pOld, pField->name) > 0) { + if (mndFindSuperTableTagIndex(pOld, pField->name) >= 0) { terrno = TSDB_CODE_MND_TAG_ALREADY_EXIST; return -1; } @@ -1193,32 +1203,39 @@ static int32_t mndAlterStb(SMnode *pMnode, SNodeMsg *pReq, const SMAlterStbReq * int32_t code = -1; STrans *pTrans = NULL; - SField *pField0 = taosArrayGet(pAlter->pFields, 0); + SField *pField0 = NULL; switch (pAlter->alterType) { case TSDB_ALTER_TABLE_ADD_TAG: code = mndAddSuperTableTag(pOld, &stbObj, pAlter->pFields, pAlter->numOfFields); break; case TSDB_ALTER_TABLE_DROP_TAG: + pField0 = taosArrayGet(pAlter->pFields, 0); code = mndDropSuperTableTag(pOld, &stbObj, pField0->name); break; case TSDB_ALTER_TABLE_UPDATE_TAG_NAME: code = mndAlterStbTagName(pOld, &stbObj, pAlter->pFields); break; case TSDB_ALTER_TABLE_UPDATE_TAG_BYTES: + pField0 = taosArrayGet(pAlter->pFields, 0); code = mndAlterStbTagBytes(pOld, &stbObj, pField0); break; case TSDB_ALTER_TABLE_ADD_COLUMN: code = mndAddSuperTableColumn(pOld, &stbObj, pAlter->pFields, pAlter->numOfFields); break; case TSDB_ALTER_TABLE_DROP_COLUMN: + pField0 = taosArrayGet(pAlter->pFields, 0); code = mndDropSuperTableColumn(pOld, &stbObj, pField0->name); break; case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: + pField0 = taosArrayGet(pAlter->pFields, 0); code = mndAlterStbColumnBytes(pOld, &stbObj, pField0); break; + case TSDB_ALTER_TABLE_UPDATE_OPTIONS: + code = mndUpdateStbComment(pOld, &stbObj, pAlter->comment, pAlter->commentLen); + break; default: - terrno = TSDB_CODE_MND_INVALID_STB_OPTION; + terrno = TSDB_CODE_OPS_NOT_SUPPORT; break; } diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 2a81f28edd..5ad4863322 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -787,6 +787,8 @@ static int32_t mndRetrieveSubscribe(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock } } + // do not show for cleared subscription +#if 0 int32_t sz = taosArrayGetSize(pSub->unassignedVgs); for (int32_t i = 0; i < sz; i++) { SMqVgEp *pVgEp = taosArrayGetP(pSub->unassignedVgs, i); @@ -829,6 +831,7 @@ static int32_t mndRetrieveSubscribe(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock numOfRows++; } +#endif taosRUnLockLatch(&pSub->lock); sdbRelease(pSdb, pSub); } diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index b62de0e06e..41d4d5f406 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -89,6 +89,8 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { SDB_SET_INT8(pRaw, dataPos, pTopic->withTbName, TOPIC_ENCODE_OVER); SDB_SET_INT8(pRaw, dataPos, pTopic->withSchema, TOPIC_ENCODE_OVER); SDB_SET_INT8(pRaw, dataPos, pTopic->withTag, TOPIC_ENCODE_OVER); + + SDB_SET_INT32(pRaw, dataPos, pTopic->consumerCnt, TOPIC_ENCODE_OVER); SDB_SET_INT32(pRaw, dataPos, pTopic->sqlLen, TOPIC_ENCODE_OVER); SDB_SET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_ENCODE_OVER); SDB_SET_INT32(pRaw, dataPos, pTopic->astLen, TOPIC_ENCODE_OVER); @@ -152,6 +154,8 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { SDB_GET_INT8(pRaw, dataPos, &pTopic->withSchema, TOPIC_DECODE_OVER); SDB_GET_INT8(pRaw, dataPos, &pTopic->withTag, TOPIC_DECODE_OVER); + SDB_GET_INT32(pRaw, dataPos, &pTopic->consumerCnt, TOPIC_DECODE_OVER); + SDB_GET_INT32(pRaw, dataPos, &pTopic->sqlLen, TOPIC_DECODE_OVER); pTopic->sql = taosMemoryCalloc(pTopic->sqlLen, sizeof(char)); if (pTopic->sql == NULL) { diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 0b4d49ed58..46dc417c6a 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -507,7 +507,7 @@ static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) { } if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_READ_DB || alterReq.alterType == TSDB_ALTER_USER_REMOVE_ALL_DB) { - if (strcmp(alterReq.dbname, "*") != 0) { + if (strcmp(alterReq.dbname, "1.*") != 0) { int32_t len = strlen(alterReq.dbname) + 1; SDbObj *pDb = mndAcquireDb(pMnode, alterReq.dbname); if (pDb == NULL) { @@ -521,7 +521,7 @@ static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) { } if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_WRITE_DB || alterReq.alterType == TSDB_ALTER_USER_REMOVE_ALL_DB) { - if (strcmp(alterReq.dbname, "*") != 0) { + if (strcmp(alterReq.dbname, "1.*") != 0) { int32_t len = strlen(alterReq.dbname) + 1; SDbObj *pDb = mndAcquireDb(pMnode, alterReq.dbname); if (pDb == NULL) { diff --git a/source/dnode/mnode/impl/test/user/user.cpp b/source/dnode/mnode/impl/test/user/user.cpp index 3c15bc3492..9e4bd79274 100644 --- a/source/dnode/mnode/impl/test/user/user.cpp +++ b/source/dnode/mnode/impl/test/user/user.cpp @@ -241,7 +241,7 @@ TEST_F(MndTestUser, 03_Alter_User) { alterReq.alterType = TSDB_ALTER_USER_REMOVE_ALL_DB; strcpy(alterReq.user, "u3"); strcpy(alterReq.pass, "1"); - strcpy(alterReq.dbname, "*"); + strcpy(alterReq.dbname, "1.*"); int32_t contLen = tSerializeSAlterUserReq(NULL, 0, &alterReq); void* pReq = rpcMallocCont(contLen); @@ -257,7 +257,7 @@ TEST_F(MndTestUser, 03_Alter_User) { alterReq.alterType = TSDB_ALTER_USER_REMOVE_ALL_DB; strcpy(alterReq.user, "u3"); strcpy(alterReq.pass, "1"); - strcpy(alterReq.dbname, "*"); + strcpy(alterReq.dbname, "1.*"); int32_t contLen = tSerializeSAlterUserReq(NULL, 0, &alterReq); void* pReq = rpcMallocCont(contLen); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 6ca60945cd..b49c148ac5 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -66,9 +66,9 @@ static void tdSRowDemo() { SRowBuilder rb = {0}; SSchema schema[DEMO_N_COLS] = { - {.type = TSDB_DATA_TYPE_TIMESTAMP, .colId = 1, .name = "ts", .bytes = 8, .flags = SCHEMA_SMA_ON}, - {.type = TSDB_DATA_TYPE_INT, .colId = 2, .name = "c1", .bytes = 4, .flags = SCHEMA_SMA_ON}, - {.type = TSDB_DATA_TYPE_INT, .colId = 3, .name = "c2", .bytes = 4, .flags = SCHEMA_SMA_ON}}; + {.type = TSDB_DATA_TYPE_TIMESTAMP, .colId = 1, .name = "ts", .bytes = 8, .flags = COL_SMA_ON}, + {.type = TSDB_DATA_TYPE_INT, .colId = 2, .name = "c1", .bytes = 4, .flags = COL_SMA_ON}, + {.type = TSDB_DATA_TYPE_INT, .colId = 3, .name = "c2", .bytes = 4, .flags = COL_SMA_ON}}; SSchema* pSchema = schema; STSchema* pTSChema = tdGetSTSChemaFromSSChema(&pSchema, numOfCols); @@ -401,7 +401,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) { fetchOffset = walGetFirstVer(pTq->pWal); } else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__LATEST) { - fetchOffset = walGetLastVer(pTq->pWal); + fetchOffset = walGetCommittedVer(pTq->pWal); } else { fetchOffset = pReq->currentOffset + 1; } diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index 3168ff53f6..2acca738fb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -38,7 +38,7 @@ struct SMemTable { struct SMemSkipListNode { int8_t level; - SMemSkipListNode *forwards[]; + SMemSkipListNode *forwards[1]; // Windows does not allow 0 }; struct SMemSkipList { @@ -46,7 +46,7 @@ struct SMemSkipList { int8_t maxLevel; int8_t level; int32_t size; - SMemSkipListNode pHead[]; + SMemSkipListNode pHead[1]; // Windows does not allow 0 }; struct SMemData { diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 10cdd9e85a..ccaef3d433 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -392,7 +392,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, tEncoderClear(&encoder); _exit: - taosArrayClear(rsp.pArray); + taosArrayDestroy(rsp.pArray); tDecoderClear(&decoder); tEncoderClear(&encoder); return rcode; @@ -454,6 +454,7 @@ static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, in SVDropTbBatchReq req = {0}; SVDropTbBatchRsp rsp = {0}; SDecoder decoder = {0}; + SEncoder encoder = {0}; int ret; pRsp->msgType = TDMT_VND_DROP_TABLE_RSP; @@ -471,7 +472,7 @@ static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, in } // process req - rsp.pArray = taosArrayInit(sizeof(SVDropTbRsp), req.nReqs); + rsp.pArray = taosArrayInit(req.nReqs, sizeof(SVDropTbRsp)); for (int iReq = 0; iReq < req.nReqs; iReq++) { SVDropTbReq *pDropTbReq = req.pReqs + iReq; SVDropTbRsp dropTbRsp = {0}; @@ -493,7 +494,11 @@ static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, in _exit: tDecoderClear(&decoder); - // encode rsp (TODO) + tEncodeSize(tEncodeSVDropTbBatchRsp, &rsp, pRsp->contLen, ret); + pRsp->pCont = rpcMallocCont(pRsp->contLen); + tEncoderInit(&encoder, pRsp->pCont, pRsp->contLen); + tEncodeSVDropTbBatchRsp(&encoder, &rsp); + tEncoderClear(&encoder); return 0; } @@ -539,6 +544,27 @@ static int vnodeDebugPrintSubmitMsg(SVnode *pVnode, SSubmitReq *pMsg, const char while (true) { if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; if (pBlock == NULL) break; + tInitSubmitBlkIter(&msgIter, pBlock, &blkIter); + if (blkIter.row == NULL) continue; + if (!pSchema || (suid != msgIter.suid)) { + if (pSchema) { + taosMemoryFreeClear(pSchema); + } + pSchema = metaGetTbTSchema(pMeta, msgIter.suid, 0); // TODO: use the real schema + if (pSchema) { + suid = msgIter.suid; + } + } + if (!pSchema) { + printf("%s:%d no valid schema\n", tags, __LINE__); + continue; + } + char __tags[128] = {0}; + snprintf(__tags, 128, "%s: uid %" PRIi64 " ", tags, msgIter.uid); + while ((row = tGetSubmitBlkNext(&blkIter))) { + tdSRowPrint(row, pSchema, __tags); + } + } vnodeDebugPrintSingleSubmitMsg(pMeta, pBlock, &msgIter, tags); } diff --git a/source/libs/executor/src/indexoperator.c b/source/libs/executor/src/indexoperator.c index 86a28605b2..fe30ebb2ea 100644 --- a/source/libs/executor/src/indexoperator.c +++ b/source/libs/executor/src/indexoperator.c @@ -21,7 +21,9 @@ typedef struct SIFCtx { int32_t code; - SHashObj *pRes; /* element is SScalarParam */ + SHashObj *pRes; /* element is SScalarParam */ + bool noExec; // true: just iterate condition tree, and add hint to executor plan + // SIdxFltStatus st; } SIFCtx; #define SIF_ERR_RET(c) \ @@ -55,11 +57,12 @@ typedef struct SIFParam { SArray *result; char * condValue; - uint8_t colValType; - col_id_t colId; - int64_t suid; // add later - char dbName[TSDB_DB_NAME_LEN]; - char colName[TSDB_COL_NAME_LEN]; + SIdxFltStatus status; + uint8_t colValType; + col_id_t colId; + int64_t suid; // add later + char dbName[TSDB_DB_NAME_LEN]; + char colName[TSDB_COL_NAME_LEN]; } SIFParam; static int32_t sifGetFuncFromSql(EOperatorType src, EIndexQueryType *dst) { @@ -82,6 +85,9 @@ static int32_t sifGetFuncFromSql(EOperatorType src, EIndexQueryType *dst) { } typedef int32_t (*sif_func_t)(SIFParam *left, SIFParam *rigth, SIFParam *output); + +static sif_func_t sifNullFunc = NULL; +// typedef struct SIFWalkParm // construct tag filter operator later static void destroyTagFilterOperatorInfo(void *param) { STagFilterOperatorInfo *pInfo = (STagFilterOperatorInfo *)param; @@ -114,6 +120,24 @@ static int32_t sifValidateColumn(SColumnNode *cn) { return TSDB_CODE_SUCCESS; } +static SIdxFltStatus sifMergeCond(ELogicConditionType type, SIdxFltStatus ls, SIdxFltStatus rs) { + // enh rule later + if (type == LOGIC_COND_TYPE_AND) { + if (ls == SFLT_NOT_INDEX || rs == SFLT_NOT_INDEX) { + if (ls == SFLT_NOT_INDEX) + return rs; + else + return ls; + } + return SFLT_COARSE_INDEX; + } else if (type == LOGIC_COND_TYPE_OR) { + return SFLT_COARSE_INDEX; + } else if (type == LOGIC_COND_TYPE_NOT) { + return SFLT_NOT_INDEX; + } + return SFLT_NOT_INDEX; +} + static int32_t sifGetValueFromNode(SNode *node, char **value) { // covert data From snode; SValueNode *vn = (SValueNode *)node; @@ -257,11 +281,11 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP if (tm == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } - SIndexMultiTermQuery *mtm = indexMultiTermQueryCreate(MUST); EIndexQueryType qtype = 0; SIF_ERR_RET(sifGetFuncFromSql(operType, &qtype)); + SIndexMultiTermQuery *mtm = indexMultiTermQueryCreate(MUST); indexMultiTermQueryAdd(mtm, tm, qtype); int ret = indexSearch(NULL, mtm, output->result); indexMultiTermQueryDestroy(mtm); @@ -319,6 +343,7 @@ static int32_t sifNotMatchFunc(SIFParam *left, SIFParam *right, SIFParam *output int id = OP_TYPE_NMATCH; return sifDoIndex(left, right, id, output); } + static int32_t sifDefaultFunc(SIFParam *left, SIFParam *right, SIFParam *output) { // add more except return TSDB_CODE_QRY_INVALID_INPUT; @@ -352,9 +377,9 @@ static sif_func_t sifGetOperFn(int32_t funcId) { case OP_TYPE_NMATCH: return sifNotMatchFunc; default: - return sifDefaultFunc; + return sifNullFunc; } - return sifDefaultFunc; + return sifNullFunc; } static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { int32_t code = 0; @@ -367,6 +392,11 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { SIF_ERR_RET(sifInitOperParams(¶ms, node, ctx)); sif_func_t operFn = sifGetOperFn(node->opType); + if (ctx->noExec && operFn == NULL) { + output->status = SFLT_NOT_INDEX; + } else { + output->status = SFLT_ACCURATE_INDEX; + } return operFn(¶ms[0], nParam > 1 ? ¶ms[1] : NULL, output); _return: @@ -385,14 +415,20 @@ static int32_t sifExecLogic(SLogicConditionNode *node, SIFCtx *ctx, SIFParam *ou SIFParam *params = NULL; SIF_ERR_RET(sifInitParamList(¶ms, node->pParameterList, ctx)); - for (int32_t m = 0; m < node->pParameterList->length; m++) { - // add impl later - if (node->condType == LOGIC_COND_TYPE_AND) { - taosArrayAddAll(output->result, params[m].result); - } else if (node->condType == LOGIC_COND_TYPE_OR) { - taosArrayAddAll(output->result, params[m].result); - } else if (node->condType == LOGIC_COND_TYPE_NOT) { - taosArrayAddAll(output->result, params[m].result); + if (ctx->noExec == false) { + for (int32_t m = 0; m < node->pParameterList->length; m++) { + // add impl later + if (node->condType == LOGIC_COND_TYPE_AND) { + taosArrayAddAll(output->result, params[m].result); + } else if (node->condType == LOGIC_COND_TYPE_OR) { + taosArrayAddAll(output->result, params[m].result); + } else if (node->condType == LOGIC_COND_TYPE_NOT) { + taosArrayAddAll(output->result, params[m].result); + } + } + } else { + for (int32_t m = 0; m < node->pParameterList->length; m++) { + output->status = sifMergeCond(node->condType, output->status, params[m].status); } } _return: @@ -486,7 +522,7 @@ static int32_t sifCalculate(SNode *pNode, SIFParam *pDst) { return TSDB_CODE_QRY_INVALID_INPUT; } int32_t code = 0; - SIFCtx ctx = {.code = 0}; + SIFCtx ctx = {.code = 0, .noExec = false}; ctx.pRes = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == ctx.pRes) { qError("index-filter failed to taosHashInit"); @@ -510,6 +546,36 @@ static int32_t sifCalculate(SNode *pNode, SIFParam *pDst) { SIF_RET(code); } +static int32_t sifGetFltHint(SNode *pNode, SIdxFltStatus *status) { + int32_t code = TSDB_CODE_SUCCESS; + if (pNode == NULL) { + return TSDB_CODE_QRY_INVALID_INPUT; + } + + SIFCtx ctx = {.code = 0, .noExec = true}; + ctx.pRes = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + if (NULL == ctx.pRes) { + qError("index-filter failed to taosHashInit"); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + nodesWalkExprPostOrder(pNode, sifCalcWalker, &ctx); + + SIF_ERR_RET(ctx.code); + + SIFParam *res = (SIFParam *)taosHashGet(ctx.pRes, (void *)&pNode, POINTER_BYTES); + if (res == NULL) { + qError("no valid res in hash, node:(%p), type(%d)", (void *)&pNode, nodeType(pNode)); + SIF_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + *status = res->status; + + sifFreeParam(res); + taosHashRemove(ctx.pRes, (void *)&pNode, POINTER_BYTES); + + SIF_RET(code); +} + int32_t doFilterTag(const SNode *pFilterNode, SArray *result) { if (pFilterNode == NULL) { return TSDB_CODE_SUCCESS; @@ -528,10 +594,14 @@ int32_t doFilterTag(const SNode *pFilterNode, SArray *result) { } SIdxFltStatus idxGetFltStatus(SNode *pFilterNode) { + SIdxFltStatus st = SFLT_NOT_INDEX; if (pFilterNode == NULL) { return SFLT_NOT_INDEX; } + SFilterInfo *filter = NULL; + // todo move to the initialization function + SIF_ERR_RET(filterInitFromNode((SNode *)pFilterNode, &filter, 0)); - // impl later - return SFLT_ACCURATE_INDEX; + SIF_ERR_RET(sifGetFltHint((SNode *)pFilterNode, &st)); + return st; } diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 27d765a441..f5ee29603e 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -93,6 +93,7 @@ int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); bool getStateFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool stateFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); int32_t stateCountFunction(SqlFunctionCtx* pCtx); +int32_t stateDurationFunction(SqlFunctionCtx* pCtx); bool getSelectivityFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 32b2f0da35..4928f75131 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -263,7 +263,7 @@ static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t l return TSDB_CODE_SUCCESS; } -static int32_t translateState(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { +static int32_t translateStateCount(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { if (3 != LIST_LENGTH(pFunc->pParameterList)) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } @@ -283,6 +283,31 @@ static int32_t translateState(SFunctionNode* pFunc, char* pErrBuf, int32_t len) return TSDB_CODE_SUCCESS; } +static int32_t translateStateDuration(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + int32_t paraNum = LIST_LENGTH(pFunc->pParameterList); + if (3 != paraNum && 4 != paraNum) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } + + uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + if (!IS_NUMERIC_TYPE(colType)) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + if (((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type != TSDB_DATA_TYPE_BINARY || + (((SExprNode*)nodesListGetNode(pFunc->pParameterList, 2))->resType.type != TSDB_DATA_TYPE_BIGINT && + ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 2))->resType.type != TSDB_DATA_TYPE_DOUBLE)) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + if (paraNum == 4 && ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 3))->resType.type != TSDB_DATA_TYPE_BIGINT) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT }; + return TSDB_CODE_SUCCESS; +} + static int32_t translateLastRow(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { // todo return TSDB_CODE_SUCCESS; @@ -701,12 +726,22 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .name = "state_count", .type = FUNCTION_TYPE_STATE_COUNT, .classification = FUNC_MGT_NONSTANDARD_SQL_FUNC, - .translateFunc = translateState, + .translateFunc = translateStateCount, .getEnvFunc = getStateFuncEnv, .initFunc = functionSetup, .processFunc = stateCountFunction, .finalizeFunc = NULL }, + { + .name = "state_duration", + .type = FUNCTION_TYPE_STATE_DURATION, + .classification = FUNC_MGT_NONSTANDARD_SQL_FUNC | FUNC_MGT_TIMELINE_FUNC, + .translateFunc = translateStateDuration, + .getEnvFunc = getStateFuncEnv, + .initFunc = functionSetup, + .processFunc = stateDurationFunction, + .finalizeFunc = NULL + }, { .name = "abs", .type = FUNCTION_TYPE_ABS, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 4d594372db..aed576a687 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -125,7 +125,10 @@ typedef enum { } EHistoBinType; typedef struct SStateInfo { - int64_t count; + union { + int64_t count; + int64_t durationStart; + }; } SStateInfo; typedef enum { @@ -2844,3 +2847,52 @@ int32_t stateCountFunction(SqlFunctionCtx* pCtx) { return numOfElems; } + +int32_t stateDurationFunction(SqlFunctionCtx* pCtx) { + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + SStateInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); + + SInputColumnInfoData* pInput = &pCtx->input; + TSKEY* tsList = (int64_t*)pInput->pPTS->pData; + + SColumnInfoData* pInputCol = pInput->pData[0]; + SColumnInfoData* pTsOutput = pCtx->pTsOutput; + + int32_t numOfElems = 0; + SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput; + + //TODO: process timeUnit for different db precisions + int32_t timeUnit = 1000; + if (pCtx->numOfParams == 5) { //TODO: param number incorrect + timeUnit = pCtx->param[3].param.i; + } + + int8_t op = getStateOpType(varDataVal(pCtx->param[1].param.pz)); + if (STATE_OPER_INVALID == op) { + return 0; + } + + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { + numOfElems++; + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + colDataAppendNULL(pOutput, i); + continue; + } + + bool ret = checkStateOp(op, pInputCol, i, pCtx->param[2].param); + int64_t output = -1; + if (ret) { + if (pInfo->durationStart == 0) { + output = 0; + pInfo->durationStart = tsList[i]; + } else { + output = (tsList[i] - pInfo->durationStart) / timeUnit; + } + } else { + pInfo->durationStart = 0; + } + colDataAppend(pOutput, i, (char *)&output, false); + } + + return numOfElems; +} diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index cdcb2592a7..5b984d766c 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2138,7 +2138,7 @@ static int32_t columnDefNodeToField(SNodeList* pList, SArray** pArray) { SField field = {.type = pCol->dataType.type, .bytes = calcTypeBytes(pCol->dataType)}; strcpy(field.name, pCol->colName); if (pCol->sma) { - field.flags |= SCHEMA_SMA_ON; + field.flags |= COL_SMA_ON; } taosArrayPush(*pArray, &field); } @@ -2321,7 +2321,7 @@ static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt static void toSchema(const SColumnDefNode* pCol, col_id_t colId, SSchema* pSchema) { int8_t flags = 0; if (pCol->sma) { - flags |= SCHEMA_SMA_ON; + flags |= COL_SMA_ON; } pSchema->colId = colId; pSchema->type = pCol->dataType.type; diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp index cf364aba5c..80921f59a9 100644 --- a/source/libs/parser/test/parInitialCTest.cpp +++ b/source/libs/parser/test/parInitialCTest.cpp @@ -117,7 +117,7 @@ TEST_F(ParserInitialCTest, createStable) { }; auto addFieldToCreateStbReqFunc = [&](bool col, const char* pFieldName, uint8_t type, int32_t bytes = 0, - int8_t flags = SCHEMA_SMA_ON) { + int8_t flags = COL_SMA_ON) { SField field = {0}; strcpy(field.name, pFieldName); field.type = type; diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 66c85b9cbb..3ee952fcdf 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -60,7 +60,9 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { if (code != 0) { int32_t err = terrno; const char *errStr = tstrerror(err); - sError("walWriteWithSyncInfo error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, errno, strerror(errno)); + int32_t linuxErr = errno; + const char *linuxErrMsg = strerror(errno); + sError("walWriteWithSyncInfo error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, linuxErr, linuxErrMsg); ASSERT(0); } //assert(code == 0); @@ -79,7 +81,9 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { if (code != 0) { int32_t err = terrno; const char *errStr = tstrerror(err); - sError("walReadWithHandle error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, errno, strerror(errno)); + int32_t linuxErr = errno; + const char *linuxErrMsg = strerror(errno); + sError("walReadWithHandle error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, linuxErr, linuxErrMsg); ASSERT(0); } //assert(walReadWithHandle(pWalHandle, index) == 0); @@ -113,7 +117,9 @@ int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex) { if (code != 0) { int32_t err = terrno; const char *errStr = tstrerror(err); - sError("walRollback error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, errno, strerror(errno)); + int32_t linuxErr = errno; + const char *linuxErrMsg = strerror(errno); + sError("walRollback error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, linuxErr, linuxErrMsg); ASSERT(0); } return 0; // to avoid compiler error @@ -144,7 +150,9 @@ int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) { if (code != 0) { int32_t err = terrno; const char *errStr = tstrerror(err); - sError("walCommit error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, errno, strerror(errno)); + int32_t linuxErr = errno; + const char *linuxErrMsg = strerror(errno); + sError("walCommit error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, linuxErr, linuxErrMsg); ASSERT(0); } return 0; // to avoid compiler error diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 3e6663f464..9aa848a7bb 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -25,6 +25,8 @@ int64_t FORCE_INLINE walGetSnaphostVer(SWal* pWal) { return pWal->vers.snapshotV int64_t FORCE_INLINE walGetLastVer(SWal* pWal) { return pWal->vers.lastVer; } +int64_t FORCE_INLINE walGetCommittedVer(SWal* pWal) { return pWal->vers.commitVer; } + static FORCE_INLINE int walBuildMetaName(SWal* pWal, int metaVer, char* buf) { return sprintf(buf, "%s/meta-ver%d", pWal->path, metaVer); } diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index e14515286e..0cfe75bf33 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -143,7 +143,11 @@ void walSetReaderCapacity(SWalReadHandle *pRead, int32_t capacity) { pRead->capa int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalHead *pHead) { int32_t code; + // TODO: valid ver + if (ver > pRead->pWal->vers.commitVer) { + return -1; + } if (pRead->curVersion != ver) { code = walReadSeekVer(pRead, ver); diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index 3400f8c516..7c877b463a 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -103,7 +103,8 @@ Dwarf_Debug tDbg; static TdThreadOnce traceThreadInit = PTHREAD_ONCE_INIT; void endTrace() { - if (traceThreadInit != PTHREAD_ONCE_INIT) { + TdThreadOnce tmp = PTHREAD_ONCE_INIT; + if (memcmp(&traceThreadInit, &tmp, sizeof(TdThreadOnce)) != 0) { delete_lookup_table(&lookup_table); dwarf_finish(tDbg); } diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c index 575d5bc187..872d8e740c 100644 --- a/source/os/src/osTimezone.c +++ b/source/os/src/osTimezone.c @@ -125,7 +125,7 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { strcpy(outTimezoneStr, tz); } -#else +#elif defined(_TD_DARWIN_64) char buf[4096] = {0}; char *tz = NULL; { @@ -170,5 +170,110 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { */ snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (%s, %+03ld00)", tz, tm1.tm_isdst ? tzname[daylight] : tzname[0], -timezone / 3600); +#else + + char buf[4096] = {0}; + char *tz = NULL; + { + int n = readlink("/etc/localtime", buf, sizeof(buf)); + if (n < 0) { + printf("read /etc/localtime error, reason:%s", strerror(errno)); + + if (taosCheckExistFile("/etc/timezone")) { + /* + * NOTE: do not remove it. + * Enforce set the correct daylight saving time(DST) flag according + * to current time + */ + time_t tx1 = taosGetTimestampSec(); + struct tm tm1; + taosLocalTime(&tx1, &tm1); + /* load time zone string from /etc/timezone */ + // FILE *f = fopen("/etc/timezone", "r"); + errno = 0; + TdFilePtr pFile = taosOpenFile("/etc/timezone", TD_FILE_READ); + char buf[68] = {0}; + if (pFile != NULL) { + int len = taosReadFile(pFile, buf, 64); + if (len < 64 && taosGetErrorFile(pFile)) { + taosCloseFile(&pFile); + printf("read /etc/timezone error, reason:%s", strerror(errno)); + return; + } + + taosCloseFile(&pFile); + + buf[sizeof(buf) - 1] = 0; + char *lineEnd = strstr(buf, "\n"); + if (lineEnd != NULL) { + *lineEnd = 0; + } + + // for CentOS system, /etc/timezone does not exist. Ignore the TZ environment variables + if (strlen(buf) > 0) { + setenv("TZ", buf, 1); + } + } + // get and set default timezone + tzset(); + /* + * get CURRENT time zone. + * system current time zone is affected by daylight saving time(DST) + * + * e.g., the local time zone of London in DST is GMT+01:00, + * otherwise is GMT+00:00 + */ + int32_t tz = (-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR; + *tsTimezone = tz; + tz += daylight; + + /* + * format example: + * + * Asia/Shanghai (CST, +0800) + * Europe/London (BST, +0100) + */ + snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); + } else { + printf("There is not /etc/timezone.\n"); + } + return; + } + buf[n] = '\0'; + for (int i = n - 1; i >= 0; --i) { + if (buf[i] == '/') { + if (tz) { + tz = buf + i + 1; + break; + } + tz = buf + i + 1; + } + } + if (!tz || 0 == strchr(tz, '/')) { + printf("parsing /etc/localtime failed"); + return; + } + + setenv("TZ", tz, 1); + tzset(); + } + + /* + * NOTE: do not remove it. + * Enforce set the correct daylight saving time(DST) flag according + * to current time + */ + time_t tx1 = taosGetTimestampSec(); + struct tm tm1; + taosLocalTime(&tx1, &tm1); + + /* + * format example: + * + * Asia/Shanghai (CST, +0800) + * Europe/London (BST, +0100) + */ + snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (%s, %+03ld00)", tz, tm1.tm_isdst ? tzname[daylight] : tzname[0], + -timezone / 3600); #endif } diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index 9dcbafca7a..10a5475555 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -911,7 +911,8 @@ void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void *param1) void taosStopCacheRefreshWorker(void) { stopRefreshWorker = true; - if(cacheThreadInit != PTHREAD_ONCE_INIT) taosThreadJoin(cacheRefreshWorker, NULL); + TdThreadOnce tmp = PTHREAD_ONCE_INIT; + if (memcmp(&cacheRefreshWorker, &tmp, sizeof(TdThreadOnce)) != 0) taosThreadJoin(cacheRefreshWorker, NULL); taosArrayDestroy(pCacheArrayList); } diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 9f197b16c8..8420b91065 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -6,6 +6,8 @@ ./test.sh -f tsim/user/pass_alter.sim ./test.sh -f tsim/user/pass_len.sim ./test.sh -f tsim/user/user_len.sim +./test.sh -f tsim/user/privilege1.sim +./test.sh -f tsim/user/privilege2.sim # ---- db ./test.sh -f tsim/db/create_all_options.sim @@ -75,8 +77,10 @@ ./test.sh -f tsim/tmq/basic3Of2Cons.sim ./test.sh -f tsim/tmq/basic4Of2Cons.sim ./test.sh -f tsim/tmq/basic2Of2ConsOverlap.sim +./test.sh -f tsim/tmq/topic.sim # --- stable +./test.sh -f tsim/stable/alter1.sim ./test.sh -f tsim/stable/disk.sim ./test.sh -f tsim/stable/dnode3.sim ./test.sh -f tsim/stable/metrics.sim diff --git a/tests/script/tsim/stable/alter1.sim b/tests/script/tsim/stable/alter1.sim new file mode 100644 index 0000000000..5cee10756c --- /dev/null +++ b/tests/script/tsim/stable/alter1.sim @@ -0,0 +1,168 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print ========== create stable +sql create database db +sql use db +sql create table db.stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 float, t3 binary(16)) comment "abd" + +sql show db.stables +if $rows != 1 then + return -1 +endi +if $data[0][3] != 3 then + return -1 +endi +if $data[0][4] != 3 then + return -1 +endi +if $data[0][6] != abd then + return -1 +endi + +print ========== add column +sql_error alter table db.stb add column ts int +sql_error alter table db.stb add column c1 int +sql_error alter table db.stb add column c2 int +sql_error alter table db.stb add column t1 int +sql_error alter table db.stb add column t2 int +sql_error alter table db.stb add column t3 int +sql alter table db.stb add column c3 int +sql alter table db.stb add column c4 bigint +sql alter table db.stb add column c5 binary(12) + +sql show db.stables +if $rows != 1 then + return -1 +endi +if $data[0][3] != 6 then + return -1 +endi +if $data[0][4] != 3 then + return -1 +endi + +print ========== drop column +sql_error alter table db.stb drop column ts +sql_error alter table db.stb drop column c6 +sql_error alter table db.stb drop column c7 +sql_error alter table db.stb drop column t1 +sql_error alter table db.stb drop column t2 +sql_error alter table db.stb drop column t3 +sql alter table db.stb drop column c1 +sql alter table db.stb drop column c4 + +sql show db.stables +if $rows != 1 then + return -1 +endi +if $data[0][3] != 4 then + return -1 +endi +if $data[0][4] != 3 then + return -1 +endi + +print ========== update column +sql_error alter table db.stb MODIFY column ts binary(20) +sql_error alter table db.stb MODIFY column c6 binary(20) +sql_error alter table db.stb MODIFY column t1 binary(20) +sql_error alter table db.stb MODIFY column t3 binary(20) +sql_error alter table db.stb MODIFY column c2 binary(3) +sql alter table db.stb MODIFY column c2 binary(32) + +sql show db.stables +if $rows != 1 then + return -1 +endi +if $data[0][3] != 4 then + return -1 +endi +if $data[0][4] != 3 then + return -1 +endi + +print ========== rename column +sql_error alter table db.stb rename column ts tx +sql_error alter table db.stb rename column c2 cx + +print ========== add tag +sql_error alter table db.stb add tag ts int +sql_error alter table db.stb add tag c2 int +sql_error alter table db.stb add tag t1 int +sql_error alter table db.stb add tag t2 int +sql_error alter table db.stb add tag t3 int +sql alter table db.stb add tag t4 bigint +sql alter table db.stb add tag c1 int +sql alter table db.stb add tag t5 binary(12) + +sql show db.stables +if $rows != 1 then + return -1 +endi +#ts c2 c3 c5 +if $data[0][3] != 4 then + return -1 +endi +#t1 t2 t3 t4 c1 t5 +if $data[0][4] != 6 then + return -1 +endi + +print ========== drop tag +sql_error alter table db.stb drop tag ts +sql_error alter table db.stb drop tag c2 +sql_error alter table db.stb drop tag c3 +sql_error alter table db.stb drop tag tx +sql alter table db.stb drop tag c1 +sql alter table db.stb drop tag t5 + +sql show db.stables +if $rows != 1 then + return -1 +endi +#ts c2 c3 c5 +if $data[0][3] != 4 then + return -1 +endi +#t1 t2 t3 t4 +if $data[0][4] != 4 then + return -1 +endi + +print ========== update tag +sql_error alter table db.stb MODIFY tag ts binary(20) +sql_error alter table db.stb MODIFY tag c2 binary(20) +sql_error alter table db.stb MODIFY tag t1 binary(20) +sql_error alter table db.stb MODIFY tag tx binary(20) +sql alter table db.stb MODIFY tag t3 binary(32) + +sql show db.stables +if $rows != 1 then + return -1 +endi +if $data[0][3] != 4 then + return -1 +endi +if $data[0][4] != 4 then + return -1 +endi + +print ========== rename tag +#t1 t2 t3 t4 + +sql_error alter table db.stb rename tag ts tx +sql_error alter table db.stb rename tag c2 cx +sql alter table db.stb rename tag t1 tx + +print ========== alter common +sql alter table db.stb comment 'abcde' ; + +sql show db.stables; +if $data[0][6] != abcde then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim b/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim index 878e3d9031..d5c800b0e9 100644 --- a/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim +++ b/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim @@ -1,337 +1,347 @@ -#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 -#basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics -#basic2Of2ConsOverlap.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics -#basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics -#basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics - -# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN -# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; -# -# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). -# - -run tsim/tmq/prepareBasicEnv-1vgrp.sim - -#---- global parameters start ----# -$dbName = db -$vgroups = 1 -$stbPrefix = stb -$ctbPrefix = ctb -$ntbPrefix = ntb -$stbNum = 1 -$ctbNum = 10 -$ntbNum = 10 -$rowsPerCtb = 10 -$tstart = 1640966400000 # 2022-01-01 00:00:00.000 -#---- global parameters end ----# - -$pullDelay = 5 -$ifcheckdata = 1 -$showMsg = 1 -$showRow = 0 - -sql connect -sql use $dbName - -print == create topics from super table -sql create topic topic_stb_column as select ts, c3 from stb -sql create topic topic_stb_all as select ts, c1, c2, c3 from stb -sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb - -print == create topics from child table -sql create topic topic_ctb_column as select ts, c3 from ctb0 -sql create topic topic_ctb_all as select * from ctb0 -sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 - -print == create topics from normal table -sql create topic topic_ntb_column as select ts, c3 from ntb0 -sql create topic topic_ntb_all as select * from ntb0 -sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 - -#sql show topics -#if $rows != 9 then -# return -1 -#endi - -$keyList = ' . group.id:cgrp1 -$keyList = $keyList . ' - -$topicNum = 2 - -#=============================== start consume =============================# - - -print ================ test consume from stb -print == overlap toipcs: topic_stb_column + topic_stb_all, topic_stb_function + topic_stb_all -$topicList = ' . topic_stb_column -$topicList = $topicList . , -$topicList = $topicList . topic_stb_all -$topicList = $topicList . ' - -$consumerId = 0 -$totalMsgOfOneTopic = $ctbNum * $rowsPerCtb -$totalMsgOfStb = $totalMsgOfOneTopic * $topicNum -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - - -$topicList = ' . topic_stb_all -$topicList = $topicList . , -$topicList = $topicList . topic_stb_function -$topicList = $topicList . ' -$consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from stb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start - -print == check consume result -wait_consumer_end_from_stb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -if $rows != 2 then - sleep 1000 - goto wait_consumer_end_from_stb -endi -if $data[0][1] == 0 then - if $data[1][1] != 1 then - return -1 - endi -endi -if $data[0][1] == 1 then - if $data[1][1] != 0 then - return -1 - endi -endi - -# $data[0][2]/$data[1][2] should be between $totalMsgOfOneTopic and $totalMsgOfStb. - -if $data[0][2] < $totalMsgOfOneTopic then - return -1 -endi -if $data[0][2] > $totalMsgOfStb then - return -1 -endi -if $data[1][2] < $totalMsgOfOneTopic then - return -1 -endi -if $data[1][2] > $totalMsgOfStb then - return -1 -endi - -$totalMsgCons = $totalMsgOfOneTopic + $totalMsgOfStb -$sumOfMsgCnt = $data[0][2] + $data[1][2] -if $sumOfMsgCnt != $totalMsgCons then - return -1 -endi - -# $data[0][3]/$data[1][3] should be between $totalMsgOfOneTopic and $totalMsgOfStb. -if $data[0][3] < $totalMsgOfOneTopic then - return -1 -endi -if $data[0][3] > $totalMsgOfStb then - return -1 -endi -if $data[1][3] < $totalMsgOfOneTopic then - return -1 -endi -if $data[1][3] > $totalMsgOfStb then - return -1 -endi - -$totalMsgCons = $totalMsgOfOneTopic + $totalMsgOfStb -$sumOfRows = $data[0][3] + $data[1][3] -if $sumOfRows != $totalMsgCons then - return -1 -endi - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdbName = cdb1 -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table for ctb -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - - -print ================ test consume from ctb -print == overlap toipcs: topic_ctb_column + topic_ctb_all, topic_ctb_function + topic_ctb_all -$topicList = ' . topic_ctb_column -$topicList = $topicList . , -$topicList = $topicList . topic_ctb_all -$topicList = $topicList . ' -$consumerId = 0 - -$totalMsgOfOneTopic = $rowsPerCtb -$totalMsgOfCtb = $totalMsgOfOneTopic * $topicNum -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -$topicList = ' . topic_ctb_function -$topicList = $topicList . , -$topicList = $topicList . topic_ctb_all -$topicList = $topicList . ' -$consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from ctb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result -wait_consumer_end_from_ctb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -if $rows != 2 then - sleep 1000 - goto wait_consumer_end_from_ctb -endi -if $data[0][1] == 0 then - if $data[1][1] != 1 then - return -1 - endi -endi -if $data[0][1] == 1 then - if $data[1][1] != 0 then - return -1 - endi -endi - -# either $data[0][2] $totalMsgOfOneTopic and $data[1][2] == $totalMsgOfCtb -# or $data[0][2] $totalMsgOfCtb and $data[1][2] == $totalMsgOfOneTopic -if $data[0][2] == $totalMsgOfOneTopic then - if $data[1][2] == $totalMsgOfCtb then - goto check_ok_0 - endi -elif $data[1][2] == $totalMsgOfOneTopic then - if $data[0][2] == $totalMsgOfCtb then - goto check_ok_0 - endi -endi -return -1 -check_ok_0: - -if $data[0][3] == $totalMsgOfOneTopic then - if $data[1][3] == $totalMsgOfCtb then - goto check_ok_1 - endi -elif $data[1][3] == $totalMsgOfOneTopic then - if $data[0][3] == $totalMsgOfCtb then - goto check_ok_1 - endi -endi -return -1 -check_ok_1: - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdbName = cdb2 -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table for ntb -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - - -print ================ test consume from ntb -print == overlap toipcs: topic_ntb_column + topic_ntb_all, topic_ntb_function + topic_ntb_all -$topicList = ' . topic_ntb_column -$topicList = $topicList . , -$topicList = $topicList . topic_ntb_all -$topicList = $topicList . ' - -$consumerId = 0 -$totalMsgOfOneTopic = $rowsPerCtb -$totalMsgOfNtb = $totalMsgOfOneTopic * $topicNum -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - - -$topicList = ' . topic_ntb_function -$topicList = $topicList . , -$topicList = $topicList . topic_ntb_all -$topicList = $topicList . ' -$consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from ntb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result from ntb -wait_consumer_end_from_ntb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -if $rows != 2 then - sleep 1000 - goto wait_consumer_end_from_ntb -endi -if $data[0][1] == 0 then - if $data[1][1] != 1 then - return -1 - endi -endi -if $data[0][1] == 1 then - if $data[1][1] != 0 then - return -1 - endi -endi - -# either $data[0][2] $totalMsgOfOneTopic and $data[1][2] == $totalMsgOfNtb -# or $data[0][2] $totalMsgOfNtb and $data[1][2] == $totalMsgOfOneTopic -if $data[0][2] == $totalMsgOfOneTopic then - if $data[1][2] == $totalMsgOfNtb then - goto check_ok_2 - endi -elif $data[1][2] == $totalMsgOfOneTopic then - if $data[0][2] == $totalMsgOfNtb then - goto check_ok_2 - endi -endi -return -1 -check_ok_2: - -if $data[0][3] == $totalMsgOfOneTopic then - if $data[1][3] == $totalMsgOfNtb then - goto check_ok_3 - endi -elif $data[1][3] == $totalMsgOfOneTopic then - if $data[0][3] == $totalMsgOfNtb then - goto check_ok_3 - endi -endi -return -1 -check_ok_3: - -#------ not need stop consumer, because it exit after pull msg overthan expect msg -#system tsim/tmq/consume.sh -s stop -x SIGINT - -system sh/exec.sh -n dnode1 -s stop -x SIGINT +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +#basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics +#basic2Of2ConsOverlap.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics +#basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics +#basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics + +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# + +run tsim/tmq/prepareBasicEnv-1vgrp.sim + +#---- global parameters start ----# +$dbName = db +$vgroups = 1 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +$pullDelay = 5 +$ifcheckdata = 1 +$showMsg = 1 +$showRow = 0 + +sql connect +sql use $dbName + +print == create topics from super table +sql create topic topic_stb_column as select ts, c3 from stb +sql create topic topic_stb_all as select ts, c1, c2, c3 from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +print == create topics from child table +sql create topic topic_ctb_column as select ts, c3 from ctb0 +sql create topic topic_ctb_all as select * from ctb0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 + +print == create topics from normal table +sql create topic topic_ntb_column as select ts, c3 from ntb0 +sql create topic topic_ntb_all as select * from ntb0 +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 + +#sql show topics +#if $rows != 9 then +# return -1 +#endi + +$keyList = ' . group.id:cgrp1 +$keyList = $keyList . ' + +$topicNum = 2 + +#=============================== start consume =============================# + + +print ================ test consume from stb +print == overlap toipcs: topic_stb_column + topic_stb_all, topic_stb_function + topic_stb_all +$topicList = ' . topic_stb_column +$topicList = $topicList . , +$topicList = $topicList . topic_stb_all +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfOneTopic = $ctbNum * $rowsPerCtb +$totalMsgOfStb = $totalMsgOfOneTopic * $topicNum +$expectmsgcnt = $totalMsgOfStb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + + +$topicList = ' . topic_stb_all +$topicList = $topicList . , +$topicList = $topicList . topic_stb_function +$topicList = $topicList . ' +$consumerId = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start + +print == check consume result +wait_consumer_end_from_stb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +if $rows != 2 then + sleep 1000 + goto wait_consumer_end_from_stb +endi +if $data[0][1] == 0 then + if $data[1][1] != 1 then + return -1 + endi +endi +if $data[0][1] == 1 then + if $data[1][1] != 0 then + return -1 + endi +endi + +# $data[0][2]/$data[1][2] should be between $totalMsgOfOneTopic and $totalMsgOfStb. + +if $data[0][2] < $totalMsgOfOneTopic then + return -1 +endi +if $data[0][2] > $totalMsgOfStb then + return -1 +endi +if $data[1][2] < $totalMsgOfOneTopic then + return -1 +endi +if $data[1][2] > $totalMsgOfStb then + return -1 +endi + +$totalMsgCons = $totalMsgOfOneTopic + $totalMsgOfStb +$sumOfMsgCnt = $data[0][2] + $data[1][2] +if $sumOfMsgCnt != $totalMsgCons then + return -1 +endi + +# $data[0][3]/$data[1][3] should be between $totalMsgOfOneTopic and $totalMsgOfStb. +if $data[0][3] < $totalMsgOfOneTopic then + return -1 +endi +if $data[0][3] > $totalMsgOfStb then + return -1 +endi +if $data[1][3] < $totalMsgOfOneTopic then + return -1 +endi +if $data[1][3] > $totalMsgOfStb then + return -1 +endi + +$totalMsgCons = $totalMsgOfOneTopic + $totalMsgOfStb +$sumOfRows = $data[0][3] + $data[1][3] +if $sumOfRows != $totalMsgCons then + return -1 +endi + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdbName = cdb1 +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table for ctb +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + + +print ================ test consume from ctb +print == overlap toipcs: topic_ctb_column + topic_ctb_all, topic_ctb_function + topic_ctb_all +$topicList = ' . topic_ctb_column +$topicList = $topicList . , +$topicList = $topicList . topic_ctb_all +$topicList = $topicList . ' +$consumerId = 0 + +$totalMsgOfOneTopic = $rowsPerCtb +$totalMsgOfCtb = $totalMsgOfOneTopic * $topicNum +$expectmsgcnt = $totalMsgOfCtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + +$topicList = ' . topic_ctb_function +$topicList = $topicList . , +$topicList = $topicList . topic_ctb_all +$topicList = $topicList . ' +$consumerId = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + +print == start consumer to pull msgs from ctb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_ctb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +if $rows != 2 then + sleep 1000 + goto wait_consumer_end_from_ctb +endi +if $data[0][1] == 0 then + if $data[1][1] != 1 then + return -1 + endi +endi +if $data[0][1] == 1 then + if $data[1][1] != 0 then + return -1 + endi +endi + +# either $data[0][2] $totalMsgOfOneTopic and $data[1][2] == $totalMsgOfCtb +# or $data[0][2] $totalMsgOfCtb and $data[1][2] == $totalMsgOfOneTopic +if $data[0][2] == $totalMsgOfOneTopic then + if $data[1][2] == $totalMsgOfCtb then + goto check_ok_0 + endi +elif $data[1][2] == $totalMsgOfOneTopic then + if $data[0][2] == $totalMsgOfCtb then + goto check_ok_0 + endi +endi +return -1 +check_ok_0: + +if $data[0][3] == $totalMsgOfOneTopic then + if $data[1][3] == $totalMsgOfCtb then + goto check_ok_1 + endi +elif $data[1][3] == $totalMsgOfOneTopic then + if $data[0][3] == $totalMsgOfCtb then + goto check_ok_1 + endi +endi +return -1 +check_ok_1: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdbName = cdb2 +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table for ntb +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + + +print ================ test consume from ntb +print == overlap toipcs: topic_ntb_column + topic_ntb_all, topic_ntb_function + topic_ntb_all +$topicList = ' . topic_ntb_column +$topicList = $topicList . , +$topicList = $topicList . topic_ntb_all +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfOneTopic = $rowsPerCtb +$totalMsgOfNtb = $totalMsgOfOneTopic * $topicNum +$expectmsgcnt = $totalMsgOfNtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + + +$topicList = ' . topic_ntb_function +$topicList = $topicList . , +$topicList = $topicList . topic_ntb_all +$topicList = $topicList . ' +$consumerId = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + +print == start consumer to pull msgs from ntb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result from ntb +wait_consumer_end_from_ntb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +if $rows != 2 then + sleep 1000 + goto wait_consumer_end_from_ntb +endi +if $data[0][1] == 0 then + if $data[1][1] != 1 then + return -1 + endi +endi +if $data[0][1] == 1 then + if $data[1][1] != 0 then + return -1 + endi +endi + +# either $data[0][2] $totalMsgOfOneTopic and $data[1][2] == $totalMsgOfNtb +# or $data[0][2] $totalMsgOfNtb and $data[1][2] == $totalMsgOfOneTopic +if $data[0][2] == $totalMsgOfOneTopic then + if $data[1][2] == $totalMsgOfNtb then + goto check_ok_2 + endi +elif $data[1][2] == $totalMsgOfOneTopic then + if $data[0][2] == $totalMsgOfNtb then + goto check_ok_2 + endi +endi +return -1 +check_ok_2: + +if $data[0][3] == $totalMsgOfOneTopic then + if $data[1][3] == $totalMsgOfNtb then + goto check_ok_3 + endi +elif $data[1][3] == $totalMsgOfOneTopic then + if $data[0][3] == $totalMsgOfNtb then + goto check_ok_3 + endi +endi +return -1 +check_ok_3: + +sql select * from performance_schema.`consumers` +if $rows != 0 then + return -1 +endi + +#sql select * from performance_schema.`subscriptions` +#if $rows != 0 then +# return -1 +#endi + +#------ not need stop consumer, because it exit after pull msg overthan expect msg +#system tsim/tmq/consume.sh -s stop -x SIGINT + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/topic.sim b/tests/script/tsim/tmq/topic.sim new file mode 100644 index 0000000000..f1dbf98bb0 --- /dev/null +++ b/tests/script/tsim/tmq/topic.sim @@ -0,0 +1,93 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +#system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 +system sh/exec.sh -n dnode1 -s start + +#---- global parameters start ----# +$dbName = db +$vgroups = 1 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +sql connect +print == create database $dbName vgroups $vgroups +sql create database $dbName vgroups $vgroups + +#wait database ready +$loop_cnt = 0 +check_db_ready: +if $loop_cnt == 10 then + print ====> database not ready! + return -1 +endi +sql show databases +print ==> rows: $rows +print ==> $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12] +print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20] +if $data(db)[19] != nostrict then + sleep 100 + $loop_cnt = $loop_cnt + 1 + goto check_db_ready +endi + +sql use $dbName + + +print == create super table +sql create table $stbPrefix (ts timestamp, c1 int, c2 float, c3 binary(16)) tags (t1 int) +sql show stables +if $rows != 1 then + return -1 +endi + +print == create child table, normal table and insert data +$i = 0 +while $i < $ctbNum + $ctb = $ctbPrefix . $i + $ntb = $ntbPrefix . $i + sql create table $ctb using $stbPrefix tags( $i ) + sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(16)) + $i = $i + 1 +endw + +print == create topics from super table +sql create topic topic_stb_column as select ts, c3 from stb +sql create topic topic_stb_all as select ts, c1, c2, c3 from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +print == create topics from child table +sql create topic topic_ctb_column as select ts, c3 from ctb0 +sql create topic topic_ctb_all as select * from ctb0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 + +print == create topics from normal table +sql create topic topic_ntb_column as select ts, c3 from ntb0 +sql create topic topic_ntb_all as select * from ntb0 +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 + + +print == show topics +sql show topics +if $rows != 9 then + return -1 +endi + +print == drop topic +sql drop topic topic_stb_column +sql drop topic topic_ctb_column +sql drop topic topic_ntb_column + +print == show topics +sql show topics +if $rows != 6 then + return -1 +endi + + diff --git a/tests/script/tsim/user/privilege2.sim b/tests/script/tsim/user/privilege2.sim new file mode 100644 index 0000000000..470f167c50 --- /dev/null +++ b/tests/script/tsim/user/privilege2.sim @@ -0,0 +1,38 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print =============== show users +sql create database d1 vgroups 1; +sql create database d2 vgroups 1; +sql create database d3 vgroups 1; +sql show databases +if $rows != 5 then + return -1 +endi + +print =============== create users +sql create user user1 PASS 'taosdata' +sql create user user2 PASS 'taosdata' +sql show users +if $rows != 3 then + return -1 +endi + +sql GRANT read ON d1.* to user1; +sql GRANT write ON d2.* to user1; + +print =============== re connect +sql close +sleep 2500 +print user user1 login +sql connect user1 + +sql_error drop database d1; +sql_error drop database d2; + +sql_error create stable d1.st (ts timestamp, i int) tags (j int) +sql create stable d2.st (ts timestamp, i int) tags (j int) + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/system-test/7-tmq/basic5.py b/tests/system-test/7-tmq/basic5.py new file mode 100644 index 0000000000..99aa4e72aa --- /dev/null +++ b/tests/system-test/7-tmq/basic5.py @@ -0,0 +1,141 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +class TDTestCase: + hostname = socket.gethostname() + rpcDebugFlagVal = '143' + clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal + + updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal + + print ("===================: ", updatecfgDict) + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + #tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def create_tables(self,dbName,vgroups,stbName,ctbNum,rowsPerTbl): + tdSql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups)) + tdSql.execute("use %s" %dbName) + tdSql.execute("create table %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) + pre_create = "create table" + sql = pre_create + #tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) + for i in range(ctbNum): + sql += " %s_%d using %s tags(%d)"%(stbName,i,stbName,i+1) + if (i > 0) and (i%100 == 0): + tdSql.execute(sql) + sql = pre_create + if sql != pre_create: + tdSql.execute(sql) + + tdLog.debug("complete to create database[%s], stable[%s] and %d child tables" %(dbName, stbName, ctbNum)) + return + + def insert_data(self,dbName,stbName,ctbNum,rowsPerTbl,startTs): + tdLog.debug("start to insert data ............") + tdSql.execute("use %s" %dbName) + pre_insert = "insert into " + sql = pre_insert + + #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) + for i in range(ctbNum): + sql += " %s_%d values "%(stbName,i) + for j in range(rowsPerTbl): + sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j) + if (j > 0) and (j%2000 == 0): + tdSql.execute(sql) + sql = "insert into %s_%d values " %(stbName,i) + #end sql + if sql != pre_insert: + # print(sql) + print("sql:%s"%sql) + tdSql.execute(sql) + tdLog.debug("insert data ............ [OK]") + return + + def prepareEnv(self, **parameterDict): + print ("input parameters:") + print (parameterDict) + self.create_tables(parameterDict["dbName"],\ + parameterDict["vgroups"],\ + parameterDict["stbName"],\ + parameterDict["ctbNum"],\ + parameterDict["rowsPerTbl"]) + + self.insert_data(parameterDict["dbName"],\ + parameterDict["stbName"],\ + parameterDict["ctbNum"],\ + parameterDict["rowsPerTbl"],\ + parameterDict["startTs"]) + return + + def run(self): + tdSql.prepare() + + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + cfgPath = buildPath + "/../sim/psim/cfg" + tdLog.info("cfgPath: %s" % cfgPath) + + tdLog.printNoPrefix("======== test scenario 1: ") + tdLog.info("step 1: create database, stb, ctb and insert data") + # create and start thread + parameterDict = {'dbName': 'db', \ + 'vgroups': 1, \ + 'stbName': 'stb', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) + prepareEnvThread.start() + + # wait for data ready + prepareEnvThread.join() + + tdLog.printNoPrefix("======== test scenario 2: ") + + + tdLog.printNoPrefix("======== test scenario 3: ") + + #os.system('pkill tmq_sim') + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase())