xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

refactor: merge from 3.0

This commit is contained in:
wangmm0220 2022-04-20 17:08:52 +08:00
parent a016f1157a a87c4f090f
commit 0fb41733ea
217 changed files with 9305 additions and 80369 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
2.0/src/client/CMakeLists.txt
View File

@ -1,95 +0,0 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20)
PROJECT(TDengine)
INCLUDE_DIRECTORIES(inc)
INCLUDE_DIRECTORIES(jni)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/zlib-1.2.11/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/plugins/http/inc)
AUX_SOURCE_DIRECTORY(src SRC)
IF (TD_LINUX)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/linux)
# set the static lib name
ADD_LIBRARY(taos_static STATIC ${SRC})
TARGET_LINK_LIBRARIES(taos_static common query trpc tutil pthread m rt cJson ${VAR_TSZ})
SET_TARGET_PROPERTIES(taos_static PROPERTIES OUTPUT_NAME "taos_static")
SET_TARGET_PROPERTIES(taos_static PROPERTIES CLEAN_DIRECT_OUTPUT 1)
# generate dynamic library (*.so)
ADD_LIBRARY(taos SHARED ${SRC})
TARGET_LINK_LIBRARIES(taos common query trpc tutil pthread m rt cJson)
IF (TD_LINUX_64)
TARGET_LINK_LIBRARIES(taos lua)
ENDIF ()
SET_TARGET_PROPERTIES(taos PROPERTIES CLEAN_DIRECT_OUTPUT 1)
#set version of .so
#VERSION so version
#SOVERSION api version
#MESSAGE(STATUS "build version ${TD_VER_NUMBER}")
SET_TARGET_PROPERTIES(taos PROPERTIES VERSION ${TD_VER_NUMBER} SOVERSION 1)
ADD_SUBDIRECTORY(tests)
ELSEIF (TD_DARWIN)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/linux)
# set the static lib name
ADD_LIBRARY(taos_static STATIC ${SRC})
TARGET_LINK_LIBRARIES(taos_static common query trpc tutil pthread m lua cJson)
SET_TARGET_PROPERTIES(taos_static PROPERTIES OUTPUT_NAME "taos_static")
SET_TARGET_PROPERTIES(taos_static PROPERTIES CLEAN_DIRECT_OUTPUT 1)
# generate dynamic library (*.dylib)
ADD_LIBRARY(taos SHARED ${SRC})
TARGET_LINK_LIBRARIES(taos common query trpc tutil pthread m lua cJson)
SET_TARGET_PROPERTIES(taos PROPERTIES CLEAN_DIRECT_OUTPUT 1)
#set version of .dylib
#VERSION dylib version
#SOVERSION dylib version
#MESSAGE(STATUS "build version ${TD_VER_NUMBER}")
SET_TARGET_PROPERTIES(taos PROPERTIES VERSION ${TD_VER_NUMBER} SOVERSION 1)
ADD_SUBDIRECTORY(tests)
ELSEIF (TD_WINDOWS)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/windows)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/windows/win32)
CONFIGURE_FILE("${TD_COMMUNITY_DIR}/src/client/src/taos.rc.in" "${TD_COMMUNITY_DIR}/src/client/src/taos.rc")
ADD_LIBRARY(taos_static STATIC ${SRC})
TARGET_LINK_LIBRARIES(taos_static trpc tutil query cJson)
# generate dynamic library (*.dll)
ADD_LIBRARY(taos SHARED ${SRC} ${TD_COMMUNITY_DIR}/src/client/src/taos.rc)
IF (NOT TD_GODLL)
SET_TARGET_PROPERTIES(taos PROPERTIES LINK_FLAGS /DEF:${TD_COMMUNITY_DIR}/src/client/src/taos.def)
ENDIF ()
TARGET_LINK_LIBRARIES(taos trpc tutil query lua cJson)
ELSEIF (TD_DARWIN)
SET(CMAKE_MACOSX_RPATH 1)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/linux)
ADD_LIBRARY(taos_static STATIC ${SRC})
TARGET_LINK_LIBRARIES(taos_static query trpc tutil pthread m lua cJson)
SET_TARGET_PROPERTIES(taos_static PROPERTIES OUTPUT_NAME "taos_static")
# generate dynamic library (*.dylib)
ADD_LIBRARY(taos SHARED ${SRC})
TARGET_LINK_LIBRARIES(taos query trpc tutil pthread m lua cJson)
SET_TARGET_PROPERTIES(taos PROPERTIES CLEAN_DIRECT_OUTPUT 1)
#set version of .so
#VERSION so version
#SOVERSION api version
#MESSAGE(STATUS "build version ${TD_VER_NUMBER}")
SET_TARGET_PROPERTIES(taos PROPERTIES VERSION ${TD_VER_NUMBER} SOVERSION 1)
ENDIF ()

85
2.0/src/client/inc/tscGlobalmerge.h
View File

@ -1,85 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TSCGLOBALMERGE_H
#define TDENGINE_TSCGLOBALMERGE_H
#ifdef __cplusplus
extern "C" {
#endif
#include "qExtbuffer.h"
#include "qFill.h"
#include "tmsg.h"
#include "tlosertree.h"
#include "qExecutor.h"
#define MAX_NUM_OF_SUBQUERY_RETRY 3
struct SQLFunctionCtx;
typedef struct SLocalDataSource {
tExtMemBuffer *pMemBuffer;
int32_t flushoutIdx;
int32_t pageId;
int32_t rowIdx;
tFilePage filePage;
} SLocalDataSource;
typedef struct SGlobalMerger {
SLocalDataSource **pLocalDataSrc;
int32_t numOfBuffer;
int32_t numOfCompleted;
int32_t numOfVnode;
SLoserTreeInfo *pLoserTree;
int32_t rowSize; // size of each intermediate result.
tOrderDescriptor *pDesc;
tExtMemBuffer **pExtMemBuffer; // disk-based buffer
char *buf; // temp buffer
} SGlobalMerger;
struct SSqlObj;
typedef struct SRetrieveSupport {
tExtMemBuffer ** pExtMemBuffer; // for build loser tree
tOrderDescriptor *pOrderDescriptor;
int32_t subqueryIndex; // index of current vnode in vnode list
struct SSqlObj *pParentSql;
tFilePage * localBuffer; // temp buffer, there is a buffer for each vnode to
uint32_t numOfRetry; // record the number of retry times
} SRetrieveSupport;
int32_t tscCreateGlobalMergerEnv(SQueryInfo* pQueryInfo, tExtMemBuffer ***pMemBuffer, int32_t numOfSub, tOrderDescriptor **pDesc, uint32_t nBufferSize, int64_t id);
void tscDestroyGlobalMergerEnv(tExtMemBuffer **pMemBuffer, tOrderDescriptor *pDesc, int32_t numOfVnodes);
int32_t saveToBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tFilePage *pPage, void *data,
int32_t numOfRows, int32_t orderType);
int32_t tscFlushTmpBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tFilePage *pPage, int32_t orderType);
/*
* create local reducer to launch the second-stage reduce process at client site
*/
int32_t tscCreateGlobalMerger(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrderDescriptor *pDesc,
SQueryInfo *pQueryInfo, SGlobalMerger **pMerger, int64_t id);
void tscDestroyGlobalMerger(SGlobalMerger* pMerger);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_TSCGLOBALMERGE_H

40
2.0/src/client/inc/tscLog.h
View File

@ -1,40 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TSCLOG_H
#define TDENGINE_TSCLOG_H
#ifdef __cplusplus
extern "C" {
#endif
#include "tlog.h"
extern uint32_t cDebugFlag;
extern int8_t tscEmbedded;
#define tscFatal(...) do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0)
#define tscError(...) do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0)
#define tscWarn(...) do { if (cDebugFlag & DEBUG_WARN) { taosPrintLog("TSC WARN ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0)
#define tscInfo(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0)
#define tscDebug(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0)
#define tscTrace(...) do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0)
#define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", cDebugFlag, __VA_ARGS__); }} while(0)
#ifdef __cplusplus
}
#endif
#endif

70
2.0/src/client/inc/tscParseLine.h
View File

@ -1,70 +0,0 @@
/*
* Copyright (c) 2021 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TSCPARSELINE_H
#define TDENGINE_TSCPARSELINE_H
#ifdef __cplusplus
extern "C" {
#endif
typedef struct {
char* key;
uint8_t type;
int16_t length;
char* value;
} TAOS_SML_KV;
typedef struct {
char* stbName;
char* childTableName;
TAOS_SML_KV* tags;
int32_t tagNum;
// first kv must be timestamp
TAOS_SML_KV* fields;
int32_t fieldNum;
} TAOS_SML_DATA_POINT;
typedef enum {
SML_TIME_STAMP_NOW,
SML_TIME_STAMP_SECONDS,
SML_TIME_STAMP_MILLI_SECONDS,
SML_TIME_STAMP_MICRO_SECONDS,
SML_TIME_STAMP_NANO_SECONDS
} SMLTimeStampType;
typedef struct {
uint64_t id;
SHashObj* smlDataToSchema;
} SSmlLinesInfo;
int tscSmlInsert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint, SSmlLinesInfo* info);
bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlLinesInfo* info);
int32_t isValidChildTableName(const char *pTbName, int16_t len);
bool convertSmlValueType(TAOS_SML_KV *pVal, char *value,
uint16_t len, SSmlLinesInfo* info);
int32_t convertSmlTimeStamp(TAOS_SML_KV *pVal, char *value,
uint16_t len, SSmlLinesInfo* info);
void destroySmlDataPoint(TAOS_SML_DATA_POINT* point);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_TSCPARSELINE_H

38
2.0/src/client/inc/tscProfile.h
View File

@ -1,38 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TSCPROFILE_H
#define TDENGINE_TSCPROFILE_H
#ifdef __cplusplus
extern "C" {
#endif
#include "tsclient.h"
void tscAddIntoSqlList(SSqlObj *pSql);
void tscRemoveFromSqlList(SSqlObj *pSql);
void tscAddIntoStreamList(SSqlStream *pStream);
void tscRemoveFromStreamList(SSqlStream *pStream, SSqlObj *pSqlObj);
int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj);
void tscKillQuery(STscObj *pObj, uint32_t killId);
void tscKillStream(STscObj *pObj, uint32_t killId);
void tscKillConnection(STscObj *pObj);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_TSCPROFILE_H

63
2.0/src/client/inc/tscSubquery.h
View File

@ -1,63 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TSCSUBQUERY_H
#define TDENGINE_TSCSUBQUERY_H
#ifdef __cplusplus
extern "C" {
#endif
#include "tscUtil.h"
#include "tsclient.h"
void tscFetchDatablockForSubquery(SSqlObj* pSql);
void tscSetupOutputColumnIndex(SSqlObj* pSql);
void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code);
SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, int32_t index);
void tscHandleMasterJoinQuery(SSqlObj* pSql);
int32_t tscHandleMasterSTableQuery(SSqlObj *pSql);
int32_t tscHandleFirstRoundStableQuery(SSqlObj *pSql);
int32_t tscHandleMultivnodeInsert(SSqlObj *pSql);
int32_t tscHandleInsertRetry(SSqlObj* parent, SSqlObj* child);
void tscBuildResFromSubqueries(SSqlObj *pSql);
TAOS_ROW doSetResultRowData(SSqlObj *pSql);
char *getArithmeticInputSrc(void *param, const char *name, int32_t colId);
void tscLockByThread(int64_t *lockedBy);
void tscUnlockByThread(int64_t *lockedBy);
int tsInsertInitialCheck(SSqlObj *pSql);
void doCleanupSubqueries(SSqlObj *pSql, int32_t numOfSubs);
void tscFreeRetrieveSup(SSqlObj *pSql);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_TSCSUBQUERY_H

381
2.0/src/client/inc/tscUtil.h
View File

@ -1,381 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TSCUTIL_H
#define TDENGINE_TSCUTIL_H
#ifdef __cplusplus
extern "C" {
#endif
#include "exception.h"
#include "os.h"
#include "qExtbuffer.h"
#include "taosdef.h"
#include "tbuffer.h"
#include "tscGlobalmerge.h"
#include "tsched.h"
#include "tsclient.h"
#define UTIL_TABLE_IS_SUPER_TABLE(metaInfo) \
(((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_SUPER_TABLE))
#define UTIL_TABLE_IS_CHILD_TABLE(metaInfo) \
(((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_CHILD_TABLE))
#define UTIL_TABLE_IS_NORMAL_TABLE(metaInfo) \
(!(UTIL_TABLE_IS_SUPER_TABLE(metaInfo) || UTIL_TABLE_IS_CHILD_TABLE(metaInfo)))
#define UTIL_TABLE_IS_TMP_TABLE(metaInfo) \
(((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_TEMP_TABLE))
#pragma pack(push,1)
// this struct is transfered as binary, padding two bytes to avoid
// an 'uid' whose low bytes is 0xff being recoginized as NULL,
// and set 'pack' to 1 to avoid break existing code.
typedef struct STidTags {
int16_t padding;
int64_t uid;
int32_t tid;
int32_t vgId;
char tag[];
} STidTags;
#pragma pack(pop)
typedef struct SJoinSupporter {
SSqlObj* pObj; // parent SqlObj
int32_t subqueryIndex; // index of sub query
SInterval interval;
SLimitVal limit; // limit info
uint64_t uid; // query table uid
SArray* colList; // previous query information, no need to use this attribute, and the corresponding attribution
SArray* exprList;
SArray* colCond;
SFieldInfo fieldsInfo;
STagCond tagCond;
SGroupbyExpr groupInfo; // group by info
struct STSBuf* pTSBuf; // the TSBuf struct that holds the compressed timestamp array
FILE* f; // temporary file in order to create TSBuf
char path[PATH_MAX]; // temporary file path, todo dynamic allocate memory
int32_t tagSize; // the length of each in the first filter stage
char* pIdTagList; // result of first stage tags
int32_t totalLen;
int32_t num;
SArray* pVgroupTables;
} SJoinSupporter;
typedef struct SMergeCtx {
SJoinSupporter* p;
int32_t idx;
SArray* res;
int8_t compared;
}SMergeCtx;
typedef struct SMergeTsCtx {
SJoinSupporter* p;
STSBuf* res;
int64_t numOfInput;
int8_t compared;
}SMergeTsCtx;
typedef struct SVgroupTableInfo {
SVgroupMsg vgInfo;
SArray *itemList; // SArray<STableIdInfo>
} SVgroupTableInfo;
typedef struct SBlockKeyTuple {
TSKEY skey;
void* payloadAddr;
} SBlockKeyTuple;
typedef struct SBlockKeyInfo {
int32_t maxBytesAlloc;
SBlockKeyTuple* pKeyTuple;
} SBlockKeyInfo;
int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len);
int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOffset, SName* name, STableMeta* pTableMeta, STableDataBlocks** dataBlocks);
void tscDestroyDataBlock(STableDataBlocks* pDataBlock, bool removeMeta);
void tscSortRemoveDataBlockDupRowsRaw(STableDataBlocks* dataBuf);
int tscSortRemoveDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKeyInfo);
int32_t tsSetBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, int32_t numOfRows);
void tscDestroyBoundColumnInfo(SParsedDataColInfo* pColInfo);
void doRetrieveSubqueryData(SSchedMsg *pMsg);
SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, int16_t bytes,
uint32_t offset);
void* tscDestroyBlockArrayList(SArray* pDataBlockList);
void* tscDestroyUdfArrayList(SArray* pUdfList);
void* tscDestroyBlockHashTable(SHashObj* pBlockHashTable, bool removeMeta);
int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock);
int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBlockMap);
int32_t tscGetDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, SName* pName, STableMeta* pTableMeta,
STableDataBlocks** dataBlocks, SArray* pBlockList);
/**
* for the projection query on metric or point interpolation query on metric,
* we iterate all the meters, instead of invoke query on all qualified meters simultaneously.
*
* @param pSql sql object
* @return
*/
bool tscIsPointInterpQuery(SQueryInfo* pQueryInfo);
bool tscIsTWAQuery(SQueryInfo* pQueryInfo);
bool tscIsIrateQuery(SQueryInfo* pQueryInfo);
bool tscIsSessionWindowQuery(SQueryInfo* pQueryInfo);
bool tscIsSecondStageQuery(SQueryInfo* pQueryInfo);
bool tsIsArithmeticQueryOnAggResult(SQueryInfo* pQueryInfo);
bool tscGroupbyColumn(SQueryInfo* pQueryInfo);
int32_t tscGetTopBotQueryExprIndex(SQueryInfo* pQueryInfo);
bool tscIsTopBotQuery(SQueryInfo* pQueryInfo);
bool hasTagValOutput(SQueryInfo* pQueryInfo);
bool timeWindowInterpoRequired(SQueryInfo *pQueryInfo);
bool isStabledev(SQueryInfo* pQueryInfo);
bool isTsCompQuery(SQueryInfo* pQueryInfo);
bool isBlockDistQuery(SQueryInfo* pQueryInfo);
bool isSimpleAggregateRv(SQueryInfo* pQueryInfo);
bool tscNonOrderedProjectionQueryOnSTable(SQueryInfo *pQueryInfo, int32_t tableIndex);
bool tscOrderedProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex);
bool tscIsDiffDerivQuery(SQueryInfo* pQueryInfo);
bool tscIsProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex);
bool tscIsProjectionQuery(SQueryInfo* pQueryInfo);
bool tscHasColumnFilter(SQueryInfo* pQueryInfo);
bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex);
bool tscQueryTags(SQueryInfo* pQueryInfo);
bool tscMultiRoundQuery(SQueryInfo* pQueryInfo, int32_t tableIndex);
bool tscQueryBlockInfo(SQueryInfo* pQueryInfo);
SExprInfo* tscAddFuncInSelectClause(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId,
SColumnIndex* pIndex, SSchema* pColSchema, int16_t colType, int16_t colId);
int32_t tscSetTableFullName(SName* pName, SStrToken* pzTableName, SSqlObj* pSql);
void tscClearInterpInfo(SQueryInfo* pQueryInfo);
bool tscIsInsertData(char* sqlstr);
// the memory is not reset in case of fast allocate payload function
int32_t tscAllocPayloadFast(SSqlCmd *pCmd, size_t size);
int32_t tscAllocPayload(SSqlCmd* pCmd, int size);
TAOS_FIELD tscCreateField(int8_t type, const char* name, int16_t bytes);
SInternalField* tscFieldInfoAppend(SFieldInfo* pFieldInfo, TAOS_FIELD* pField);
SInternalField* tscFieldInfoInsert(SFieldInfo* pFieldInfo, int32_t index, TAOS_FIELD* field);
SInternalField* tscFieldInfoGetInternalField(SFieldInfo* pFieldInfo, int32_t index);
TAOS_FIELD* tscFieldInfoGetField(SFieldInfo* pFieldInfo, int32_t index);
void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo);
int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index);
void tscFieldInfoClear(SFieldInfo* pFieldInfo);
void tscFieldInfoCopy(SFieldInfo* pFieldInfo, const SFieldInfo* pSrc, const SArray* pExprList);
static FORCE_INLINE int32_t tscNumOfFields(SQueryInfo* pQueryInfo) { return pQueryInfo->fieldsInfo.numOfOutput; }
int32_t tscGetFirstInvisibleFieldPos(SQueryInfo* pQueryInfo);
int32_t tscFieldInfoCompare(const SFieldInfo* pFieldInfo1, const SFieldInfo* pFieldInfo2, int32_t *diffSize);
void tscInsertPrimaryTsSourceColumn(SQueryInfo* pQueryInfo, uint64_t uid);
int32_t tscFieldInfoSetSize(const SFieldInfo* pFieldInfo1, const SFieldInfo* pFieldInfo2);
void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes);
int32_t tscGetResRowLength(SArray* pExprList);
SExprInfo* tscExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, SColumnIndex* pColIndex, int16_t type,
int16_t size, int16_t resColId, int16_t interSize, bool isTagCol);
SExprInfo* tscExprCreate(STableMetaInfo* pTableMetaInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type,
int16_t size, int16_t resColId, int16_t interSize, int32_t colType);
void tscExprAddParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes);
SExprInfo* tscExprAppend(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type,
int16_t size, int16_t resColId, int16_t interSize, bool isTagCol);
SExprInfo* tscExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, int16_t srcColumnIndex, int16_t type,
int16_t size);
size_t tscNumOfExprs(SQueryInfo* pQueryInfo);
int32_t tscExprTopBottomIndex(SQueryInfo* pQueryInfo);
SExprInfo *tscExprGet(SQueryInfo* pQueryInfo, int32_t index);
int32_t tscExprCopy(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy);
int32_t tscExprCopyAll(SArray* dst, const SArray* src, bool deepcopy);
void tscExprAssign(SExprInfo* dst, const SExprInfo* src);
void tscExprDestroy(SArray* pExprInfo);
int32_t createProjectionExpr(SQueryInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SExprInfo*** pExpr, int32_t* num);
void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, bool removeMeta, uint64_t id);
SColumn* tscColumnClone(const SColumn* src);
void tscColumnCopy(SColumn* pDest, const SColumn* pSrc);
int32_t tscColumnExists(SArray* pColumnList, int32_t columnId, uint64_t uid);
SColumn* tscColumnListInsert(SArray* pColumnList, int32_t columnIndex, uint64_t uid, SSchema* pSchema);
void tscColumnListDestroy(SArray* pColList);
void tscColumnListCopy(SArray* dst, const SArray* src, uint64_t tableUid);
void tscColumnListCopyAll(SArray* dst, const SArray* src);
void convertQueryResult(SSqlRes* pRes, SQueryInfo* pQueryInfo, uint64_t objId, bool convertNchar);
void tscDequoteAndTrimToken(SStrToken* pToken);
int32_t tscValidateName(SStrToken* pToken);
void tscIncStreamExecutionCount(void* pStream);
bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId, int32_t numOfParams);
// get starter position of metric query condition (query on tags) in SSqlCmd.payload
SCond* tsGetSTableQueryCond(STagCond* pCond, uint64_t uid);
void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBufferWriter* bw);
int32_t tscTagCondCopy(STagCond* dest, const STagCond* src);
int32_t tscColCondCopy(SArray** dest, const SArray* src, uint64_t uid, int16_t tidx);
void tscTagCondRelease(STagCond* pCond);
void tscColCondRelease(SArray** pCond);
void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SQueryInfo* pQueryInfo);
bool tscShouldBeFreed(SSqlObj* pSql);
STableMetaInfo* tscGetTableMetaInfoFromCmd(SSqlCmd *pCmd, int32_t tableIndex);
STableMetaInfo* tscGetMetaInfo(SQueryInfo *pQueryInfo, int32_t tableIndex);
void tscInitQueryInfo(SQueryInfo* pQueryInfo);
void tscClearSubqueryInfo(SSqlCmd* pCmd);
int32_t tscAddQueryInfo(SSqlCmd *pCmd);
SQueryInfo *tscGetQueryInfo(SSqlCmd* pCmd);
SQueryInfo *tscGetQueryInfoS(SSqlCmd *pCmd);
void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo);
STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, SName* name, STableMeta* pTableMeta,
SVgroupsInfo* vgroupList, SArray* pTagCols, SArray* pVgroupTables);
STableMetaInfo* tscAddEmptyMetaInfo(SQueryInfo *pQueryInfo);
void tscFreeVgroupTableInfo(SArray* pVgroupTables);
SArray* tscVgroupTableInfoDup(SArray* pVgroupTables);
void tscRemoveVgroupTableGroup(SArray* pVgroupTable, int32_t index);
void tscVgroupTableCopy(SVgroupTableInfo* info, SVgroupTableInfo* pInfo);
int tscGetSTableVgroupInfo(SSqlObj* pSql, SQueryInfo* pQueryInfo);
int tscGetTableMeta(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo);
int tscGetTableMetaEx(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool createIfNotExists, bool onlyLocal);
int32_t tscGetUdfFromNode(SSqlObj *pSql, SQueryInfo* pQueryInfo);
void tscResetForNextRetrieve(SSqlRes* pRes);
void executeQuery(SSqlObj* pSql, SQueryInfo* pQueryInfo);
void doExecuteQuery(SSqlObj* pSql, SQueryInfo* pQueryInfo);
SVgroupsInfo* tscVgroupInfoClone(SVgroupsInfo *pInfo);
void* tscVgroupInfoClear(SVgroupsInfo *pInfo);
#if 0
void tscSVgroupInfoCopy(SVgroupInfo* dst, const SVgroupInfo* src);
#endif
/**
* The create object function must be successful expect for the out of memory issue.
*
* Therefore, the metermeta/metricmeta object is directly passed to the newly created subquery object from the
* previous sql object, instead of retrieving the metermeta/metricmeta from cache.
*
* Because the metermeta/metricmeta may have been released by other threads, resulting in the retrieving failed as
* well as the create function.
*
* @param pSql
* @param vnodeIndex
* @param tableIndex
* @param fp
* @param param
* @param pPrevSql
* @return
*/
SSqlObj* createSimpleSubObj(SSqlObj* pSql, __async_cb_func_t fp, void* param, int32_t cmd);
void registerSqlObj(SSqlObj* pSql);
void tscInitResForMerge(SSqlRes* pRes);
SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t fp, void* param, int32_t cmd, SSqlObj* pPrevSql);
void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClauseIndex, int32_t tableIndex);
void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex, SSqlCmd* pCmd);
int16_t tscGetJoinTagColIdByUid(STagCond* pTagCond, uint64_t uid);
int16_t tscGetTagColIndexById(STableMeta* pTableMeta, int16_t colId);
int32_t doInitSubState(SSqlObj* pSql, int32_t numOfSubqueries);
void tscPrintSelNodeList(SSqlObj* pSql, int32_t subClauseIndex);
bool hasMoreVnodesToTry(SSqlObj *pSql);
bool hasMoreClauseToTry(SSqlObj* pSql);
void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeCachedMeta, uint64_t id);
void tscTryQueryNextVnode(SSqlObj *pSql, __async_cb_func_t fp);
void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp);
int tscSetMgmtEpSetFromCfg(const char *first, const char *second, SRpcCorEpSet *corEpSet);
int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVgroupNameList, SArray* pUdfList, __async_cb_func_t fp, bool metaClone);
int tscTransferTableNameList(SSqlObj *pSql, const char *pNameList, int32_t length, SArray* pNameArray);
bool subAndCheckDone(SSqlObj *pSql, SSqlObj *pParentSql, int idx);
bool tscSetSqlOwner(SSqlObj* pSql);
void tscClearSqlOwner(SSqlObj* pSql);
int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_t rowSize, int32_t finalRowSize);
char* serializeTagData(STagData* pTagData, char* pMsg);
int32_t copyTagData(STagData* dst, const STagData* src);
STableMeta* createSuperTableMeta(STableMetaMsg* pChild);
uint32_t tscGetTableMetaSize(STableMeta* pTableMeta);
CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta);
uint32_t tscGetTableMetaMaxSize();
int32_t tscCreateTableMetaFromSTableMeta(STableMeta** ppChild, const char* name, size_t *tableMetaCapacity, STableMeta **ppStable);
STableMeta* tscTableMetaDup(STableMeta* pTableMeta);
SVgroupsInfo* tscVgroupsInfoDup(SVgroupsInfo* pVgroupsInfo);
int32_t tscGetTagFilterSerializeLen(SQueryInfo* pQueryInfo);
int32_t tscGetColFilterSerializeLen(SQueryInfo* pQueryInfo);
int32_t tscCreateQueryFromQueryInfo(SQueryInfo* pQueryInfo, SQueryAttr* pQueryAttr, void* addr);
void* createQInfoFromQueryNode(SQueryInfo* pQueryInfo, STableGroupInfo* pTableGroupInfo, SOperatorInfo* pOperator, char* sql, void* addr, int32_t stage, uint64_t qId);
void* malloc_throw(size_t size);
void* calloc_throw(size_t nmemb, size_t size);
char* strdup_throw(const char* str);
bool vgroupInfoIdentical(SNewVgroupInfo *pExisted, SVgroupMsg* src);
SNewVgroupInfo createNewVgroupInfo(SVgroupMsg *pVgroupMsg);
STblCond* tsGetTableFilter(SArray* filters, uint64_t uid, int16_t idx);
void tscRemoveCachedTableMeta(STableMetaInfo* pTableMetaInfo, uint64_t id);
char* cloneCurrentDBName(SSqlObj* pSql);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_TSCUTIL_H

837
2.0/src/client/inc/tsclient.h
View File

@ -1,837 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TSCLIENT_H
#define TDENGINE_TSCLIENT_H
#ifdef __cplusplus
extern "C" {
#endif
#include "os.h"
#include "../../../include/client/taos.h"
#include "qAggMain.h"
#include "taosdef.h"
#include "tmsg.h"
#include "tarray.h"
#include "tcache.h"
#include "tglobal.h"
#include "tref.h"
#include "tutil.h"
#include "qExecutor.h"
#include "qSqlparser.h"
#include "qTsbuf.h"
#include "qUtil.h"
#include "tmsgtype.h"
typedef enum {
TAOS_REQ_FROM_SHELL,
TAOS_REQ_FROM_HTTP
} SReqOrigin;
// forward declaration
struct SSqlInfo;
typedef void (*__async_cb_func_t)(void *param, TAOS_RES *tres, int32_t numOfRows);
typedef struct SNewVgroupInfo {
int32_t vgId;
int8_t inUse;
int8_t numOfEps;
SEpAddrMsg ep[TSDB_MAX_REPLICA];
} SNewVgroupInfo;
typedef struct CChildTableMeta {
int32_t vgId;
STableId id;
uint8_t tableType;
char sTableName[TSDB_TABLE_FNAME_LEN]; // TODO: refactor super table name, not full name
uint64_t suid; // super table id
} CChildTableMeta;
typedef struct SColumnIndex {
int16_t tableIndex;
int16_t columnIndex;
} SColumnIndex;
typedef struct SColumn {
uint64_t tableUid;
int32_t columnIndex;
SColumnInfo info;
} SColumn;
typedef struct SInternalField {
TAOS_FIELD field;
bool visible;
SExprInfo *pExpr;
} SInternalField;
typedef struct SParamInfo {
int32_t idx;
uint8_t type;
uint8_t timePrec;
int16_t bytes;
uint32_t offset;
} SParamInfo;
typedef struct SBoundColumn {
int32_t offset; // all column offset value
int32_t toffset; // first part offset for SDataRow TODO: get offset from STSchema on future
uint8_t valStat; // denote if current column bound or not(0 means has val, 1 means no val)
} SBoundColumn;
typedef enum {
VAL_STAT_HAS = 0x0, // 0 means has val
VAL_STAT_NONE = 0x01, // 1 means no val
} EValStat;
typedef struct {
uint16_t schemaColIdx;
uint16_t boundIdx;
uint16_t finalIdx;
} SBoundIdxInfo;
typedef enum _COL_ORDER_STATUS {
ORDER_STATUS_UNKNOWN = 0,
ORDER_STATUS_ORDERED = 1,
ORDER_STATUS_DISORDERED = 2,
} EOrderStatus;
typedef struct SParsedDataColInfo {
int16_t numOfCols;
int16_t numOfBound;
uint16_t flen; // TODO: get from STSchema
uint16_t allNullLen; // TODO: get from STSchema(base on SDataRow)
uint16_t extendedVarLen;
uint16_t boundNullLen; // bound column len with all NULL value(without VarDataOffsetT/SColIdx part)
int32_t *boundColumns; // bound column idx according to schema
SBoundColumn * cols;
SBoundIdxInfo *colIdxInfo;
int8_t orderStatus; // bound columns
} SParsedDataColInfo;
#define IS_DATA_COL_ORDERED(spd) ((spd->orderStatus) == (int8_t)ORDER_STATUS_ORDERED)
typedef struct {
uint8_t rowType; // default is 0, that is SDataRow
int32_t rowSize;
} SMemRowBuilder;
int tsParseTime(SStrToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec);
int initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, SParsedDataColInfo *pColInfo);
void destroyMemRowBuilder(SMemRowBuilder *pBuilder);
/**
* @brief
*
* @param rowType
* @param spd
* @param idx the absolute bound index of columns
* @return FORCE_INLINE
*/
static FORCE_INLINE void tscGetSTSRowAppendInfo(SSchema *pSchema, uint8_t rowType, SParsedDataColInfo *spd, int32_t idx,
int32_t *toffset, int16_t *colId) {
int32_t schemaIdx = 0;
if (IS_DATA_COL_ORDERED(spd)) {
schemaIdx = spd->boundColumns[idx];
if (isDataRowT(rowType)) {
*toffset = (spd->cols + schemaIdx)->toffset; // the offset of firstPart
} else {
*toffset = idx * sizeof(SColIdx); // the offset of SColIdx
}
} else {
ASSERT(idx == (spd->colIdxInfo + idx)->boundIdx);
schemaIdx = (spd->colIdxInfo + idx)->schemaColIdx;
if (isDataRowT(rowType)) {
*toffset = (spd->cols + schemaIdx)->toffset;
} else {
*toffset = ((spd->colIdxInfo + idx)->finalIdx) * sizeof(SColIdx);
}
}
*colId = pSchema[schemaIdx].colId;
}
typedef struct STableDataBlocks {
SName tableName;
int8_t tsSource; // where does the UNIX timestamp come from, server or client
bool ordered; // if current rows are ordered or not
int64_t vgId; // virtual group id
int64_t prevTS; // previous timestamp, recorded to decide if the records array is ts ascending
int32_t numOfTables; // number of tables in current submit block
int32_t rowSize; // row size for current table
uint32_t nAllocSize;
uint32_t headerSize; // header for table info (uid, tid, submit metadata)
uint32_t size;
STableMeta *pTableMeta; // the tableMeta of current table, the table meta will be used during submit, keep a ref to avoid to be removed from cache
char *pData;
bool cloned;
SParsedDataColInfo boundColumnInfo;
// for parameter ('?') binding
uint32_t numOfAllocedParams;
uint32_t numOfParams;
SParamInfo * params;
SMemRowBuilder rowBuilder;
} STableDataBlocks;
typedef struct {
STableMeta *pTableMeta;
SArray *vgroupIdList;
} STableMetaVgroupInfo;
typedef struct SInsertStatementParam {
SName **pTableNameList; // all involved tableMeta list of current insert sql statement.
int32_t numOfTables; // number of tables in table name list
SHashObj *pTableBlockHashList; // data block for each table
SArray *pDataBlocks; // SArray<STableDataBlocks*>. Merged submit block for each vgroup
int8_t schemaAttached; // denote if submit block is built with table schema or not
uint8_t payloadType; // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert
STagData tagData; // NOTE: pTagData->data is used as a variant length array
int32_t batchSize; // for parameter ('?') binding and batch processing
int32_t numOfParams;
char msg[512]; // error message
uint32_t insertType; // insert data from [file|sql statement| bound statement]
uint64_t objectId; // sql object id
char *sql; // current sql statement position
} SInsertStatementParam;
typedef enum {
PAYLOAD_TYPE_KV = 0,
PAYLOAD_TYPE_RAW = 1,
} EPayloadType;
#define IS_RAW_PAYLOAD(t) \
(((int)(t)) == PAYLOAD_TYPE_RAW) // 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert
// TODO extract sql parser supporter
typedef struct {
int command;
uint8_t msgType;
SInsertStatementParam insertParam;
char reserve1[3]; // fix bus error on arm32
int32_t count; // todo remove it
bool subCmd;
char reserve2[3]; // fix bus error on arm32
int16_t numOfCols;
char reserve3[2]; // fix bus error on arm32
uint32_t allocSize;
char * payload;
int32_t payloadLen;
SHashObj *pTableMetaMap; // local buffer to keep the queried table meta, before validating the AST
SQueryInfo *pQueryInfo;
SQueryInfo *active; // current active query info
int32_t batchSize; // for parameter ('?') binding and batch processing
int32_t resColumnId;
} SSqlCmd;
typedef struct {
int32_t numOfRows; // num of results in current retrieval
int64_t numOfTotal; // num of total results
int64_t numOfClauseTotal; // num of total result in current subclause
char * pRsp;
int32_t rspType;
int32_t rspLen;
uint64_t qId; // query id of SQInfo
int64_t useconds;
int64_t offset; // offset value from vnode during projection query of stable
int32_t row;
int16_t numOfCols;
int16_t precision;
bool completed;
int32_t code;
char * data;
TAOS_ROW tsrow;
TAOS_ROW urow;
int32_t* length; // length for each field for current row
char ** buffer; // Buffer used to put multibytes encoded using unicode (wchar_t)
SColumnIndex* pColumnIndex;
TAOS_FIELD* final;
struct SGlobalMerger *pMerger;
} SSqlRes;
typedef struct {
char key[512];
void *pDnodeConn;
} SRpcObj;
typedef struct STscObj {
void * signature;
void * pTimer;
char user[TSDB_USER_LEN];
char pass[TSDB_KEY_LEN];
char acctId[TSDB_ACCT_ID_LEN];
char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN];
char sversion[TSDB_VERSION_LEN];
char writeAuth : 1;
char superAuth : 1;
uint32_t connId;
uint64_t rid; // ref ID returned by taosAddRef
int64_t hbrid;
struct SSqlObj * sqlList;
struct SSqlStream *streamList;
SRpcObj *pRpcObj;
SRpcCorEpSet *tscCorMgmtEpSet;
pthread_mutex_t mutex;
int32_t numOfObj; // number of sqlObj from this tscObj
SReqOrigin from;
} STscObj;
typedef struct SSubqueryState {
pthread_mutex_t mutex;
int8_t *states;
int32_t numOfSub; // the number of total sub-queries
uint64_t numOfRetrievedRows; // total number of points in this query
} SSubqueryState;
typedef struct SSqlObj {
void *signature;
int64_t owner; // owner of sql object, by which it is executed
STscObj *pTscObj;
int64_t rpcRid;
__async_cb_func_t fp;
__async_cb_func_t fetchFp;
void *param;
int64_t stime;
uint32_t queryId;
void * pStream;
void * pSubscription;
char * sqlstr;
void * pBuf; // table meta buffer
char parseRetry;
char retry;
char maxRetry;
SRpcEpSet epSet;
char listed;
tsem_t rspSem;
SSqlCmd cmd;
SSqlRes res;
SSubqueryState subState;
struct SSqlObj **pSubs;
int64_t metaRid;
int64_t svgroupRid;
int64_t squeryLock;
int32_t retryReason; // previous error code
struct SSqlObj *prev, *next;
int64_t self;
} SSqlObj;
typedef struct SSqlStream {
SSqlObj *pSql;
void * cqhandle; // stream belong to SCQContext handle
const char* dstTable;
uint32_t streamId;
char listed;
bool isProject;
int16_t precision;
int64_t num; // number of computing count
/*
* keep the number of current result in computing,
* the value will be set to 0 before set timer for next computing
*/
int64_t numOfRes;
int64_t useconds; // total elapsed time
int64_t ctime; // stream created time
int64_t stime; // stream next executed time
int64_t etime; // stream end query time, when time is larger then etime, the stream will be closed
int64_t ltime; // stream last row time in stream table
SInterval interval;
void * pTimer;
void (*fp)();
void *param;
void (*callback)(void *); // Callback function when stream is stopped from client level
struct SSqlStream *prev, *next;
} SSqlStream;
void tscSetStreamDestTable(SSqlStream* pStream, const char* dstTable);
int tscAcquireRpc(const char *key, const char *user, const char *secret,void **pRpcObj);
void tscReleaseRpc(void *param);
void tscInitMsgsFp();
int tsParseSql(SSqlObj *pSql, bool initial);
void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet);
int tscBuildAndSendRequest(SSqlObj *pSql, SQueryInfo* pQueryInfo);
int tscRenewTableMeta(SSqlObj *pSql, int32_t tableIndex);
void tscAsyncResultOnError(SSqlObj *pSql);
void tscQueueAsyncError(void(*fp), void *param, int32_t code);
int tscProcessLocalCmd(SSqlObj *pSql);
int tscCfgDynamicOptions(char *msg);
int32_t tscTansformFuncForSTableQuery(SQueryInfo *pQueryInfo);
void tscRestoreFuncForSTableQuery(SQueryInfo *pQueryInfo);
int32_t tscCreateResPointerInfo(SSqlRes *pRes, SQueryInfo *pQueryInfo);
void tscSetResRawPtr(SSqlRes* pRes, SQueryInfo* pQueryInfo);
void tscSetResRawPtrRv(SSqlRes* pRes, SQueryInfo* pQueryInfo, SSDataBlock* pBlock, bool convertNchar);
void handleDownstreamOperator(SSqlObj** pSqlList, int32_t numOfUpstream, SQueryInfo* px, SSqlObj* pParent);
void destroyTableNameList(SInsertStatementParam* pInsertParam);
void tscResetSqlCmd(SSqlCmd *pCmd, bool removeMeta, uint64_t id);
/**
* free query result of the sql object
* @param pObj
*/
void tscFreeSqlResult(SSqlObj *pSql);
void* tscCleanupTableMetaMap(SHashObj* pTableMetaMap);
/**
* free sql object, release allocated resource
* @param pObj
*/
void tscFreeSqlObj(SSqlObj *pSql);
void tscFreeSubobj(SSqlObj* pSql);
void tscFreeRegisteredSqlObj(void *pSql);
void tscCloseTscObj(void *pObj);
// todo move to taos? or create a new file: taos_internal.h
TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int),
void *param, TAOS **taos);
TAOS_RES* taos_query_h(TAOS* taos, const char *sqlstr, int64_t* res);
TAOS_RES * taos_query_ra(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *param);
void waitForQueryRsp(void *param, TAOS_RES *tres, int code);
void doAsyncQuery(STscObj *pObj, SSqlObj *pSql, __async_cb_func_t fp, void *param, const char *sqlstr, size_t sqlLen);
void tscImportDataFromFile(SSqlObj *pSql);
struct SGlobalMerger* tscInitResObjForLocalQuery(int32_t numOfRes, int32_t rowLen, uint64_t id);
bool tscIsUpdateQuery(SSqlObj* pSql);
char* tscGetSqlStr(SSqlObj* pSql);
bool tscIsQueryWithLimit(SSqlObj* pSql);
bool tscHasReachLimitation(SQueryInfo *pQueryInfo, SSqlRes *pRes);
void tscSetBoundColumnInfo(SParsedDataColInfo *pColInfo, SSchema *pSchema, int32_t numOfCols);
char *tscGetErrorMsgPayload(SSqlCmd *pCmd);
int32_t tscErrorMsgWithCode(int32_t code, char* dstBuffer, const char* errMsg, const char* sql);
int32_t tscInvalidOperationMsg(char *msg, const char *additionalInfo, const char *sql);
int32_t tscSQLSyntaxErrMsg(char* msg, const char* additionalInfo, const char* sql);
int32_t tscValidateSqlInfo(SSqlObj *pSql, struct SSqlInfo *pInfo);
int32_t tsSetBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, int32_t numOfRows);
extern int32_t sentinel;
extern SHashObj *tscVgroupMap;
extern SHashObj *tscTableMetaMap;
extern SCacheObj *tscVgroupListBuf;
extern int tscObjRef;
extern void *tscTmr;
extern void *tscQhandle;
extern int tscKeepConn[];
extern int tscRefId;
extern int tscNumOfObj; // number of existed sqlObj in current process.
extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo);
void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables);
int16_t getNewResColId(SSqlCmd* pCmd);
int32_t schemaIdxCompar(const void *lhs, const void *rhs);
int32_t boundIdxCompar(const void *lhs, const void *rhs);
static FORCE_INLINE int32_t getExtendedRowSize(STableDataBlocks *pBlock) {
ASSERT(pBlock->rowSize == pBlock->pTableMeta->tableInfo.rowSize);
return pBlock->rowSize + TD_MEM_ROW_DATA_HEAD_SIZE + pBlock->boundColumnInfo.extendedVarLen;
}
static FORCE_INLINE void initSMemRow(SMemRow row, uint8_t memRowType, STableDataBlocks *pBlock, int16_t nBoundCols) {
memRowSetType(row, memRowType);
if (isDataRowT(memRowType)) {
dataRowSetVersion(memRowDataBody(row), pBlock->pTableMeta->sversion);
dataRowSetLen(memRowDataBody(row), (TDRowLenT)(TD_DATA_ROW_HEAD_SIZE + pBlock->boundColumnInfo.flen));
} else {
ASSERT(nBoundCols > 0);
memRowSetKvVersion(row, pBlock->pTableMeta->sversion);
kvRowSetNCols(memRowKvBody(row), nBoundCols);
kvRowSetLen(memRowKvBody(row), (TDRowLenT)(TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nBoundCols));
}
}
/**
* TODO: Move to tdataformat.h and refactor when STSchema available.
* - fetch flen and toffset from STSChema and remove param spd
*/
static FORCE_INLINE void convertToSDataRow(SMemRow dest, SMemRow src, SSchema *pSchema, int nCols,
SParsedDataColInfo *spd) {
ASSERT(isKvRow(src));
SKVRow kvRow = memRowKvBody(src);
SDataRow dataRow = memRowDataBody(dest);
memRowSetType(dest, SMEM_ROW_DATA);
dataRowSetVersion(dataRow, memRowKvVersion(src));
dataRowSetLen(dataRow, (TDRowLenT)(TD_DATA_ROW_HEAD_SIZE + spd->flen));
int32_t kvIdx = 0;
for (int i = 0; i < nCols; ++i) {
SSchema *schema = pSchema + i;
void * val = tdGetKVRowValOfColEx(kvRow, schema->colId, &kvIdx);
tdAppendDataColVal(dataRow, val != NULL ? val : getNullValue(schema->type), true, schema->type,
(spd->cols + i)->toffset);
}
}
// TODO: Move to tdataformat.h and refactor when STSchema available.
static FORCE_INLINE void convertToSKVRow(SMemRow dest, SMemRow src, SSchema *pSchema, int nCols, int nBoundCols,
SParsedDataColInfo *spd) {
ASSERT(isDataRow(src));
SDataRow dataRow = memRowDataBody(src);
SKVRow kvRow = memRowKvBody(dest);
memRowSetType(dest, SMEM_ROW_KV);
memRowSetKvVersion(kvRow, dataRowVersion(dataRow));
kvRowSetNCols(kvRow, nBoundCols);
kvRowSetLen(kvRow, (TDRowLenT)(TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nBoundCols));
int32_t toffset = 0, kvOffset = 0;
for (int i = 0; i < nCols; ++i) {
if ((spd->cols + i)->valStat == VAL_STAT_HAS) {
SSchema *schema = pSchema + i;
toffset = (spd->cols + i)->toffset;
void *val = tdGetRowDataOfCol(dataRow, schema->type, toffset + TD_DATA_ROW_HEAD_SIZE);
tdAppendKvColVal(kvRow, val, true, schema->colId, schema->type, kvOffset);
kvOffset += sizeof(SColIdx);
}
}
}
// TODO: Move to tdataformat.h and refactor when STSchema available.
static FORCE_INLINE void convertSMemRow(SMemRow dest, SMemRow src, STableDataBlocks *pBlock) {
STableMeta * pTableMeta = pBlock->pTableMeta;
STableComInfo tinfo = tscGetTableInfo(pTableMeta);
SSchema * pSchema = tscGetTableSchema(pTableMeta);
SParsedDataColInfo *spd = &pBlock->boundColumnInfo;
ASSERT(dest != src);
if (isDataRow(src)) {
// TODO: Can we use pBlock -> numOfParam directly?
ASSERT(spd->numOfBound > 0);
convertToSKVRow(dest, src, pSchema, tinfo.numOfColumns, spd->numOfBound, spd);
} else {
convertToSDataRow(dest, src, pSchema, tinfo.numOfColumns, spd);
}
}
static bool isNullStr(SStrToken *pToken) {
return (pToken->type == TK_NULL) || ((pToken->type == TK_STRING) && (pToken->n != 0) &&
(strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0));
}
static FORCE_INLINE int32_t tscToDouble(SStrToken *pToken, double *value, char **endPtr) {
errno = 0;
*value = strtold(pToken->z, endPtr);
// not a valid integer number, return error
if ((*endPtr - pToken->z) != pToken->n) {
return TK_ILLEGAL;
}
return pToken->type;
}
static uint8_t TRUE_VALUE = (uint8_t)TSDB_TRUE;
static uint8_t FALSE_VALUE = (uint8_t)TSDB_FALSE;
static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, SMemRow row, char *msg, char **str,
bool primaryKey, int16_t timePrec, int32_t toffset, int16_t colId) {
int64_t iv;
int32_t ret;
char * endptr = NULL;
if (IS_NUMERIC_TYPE(pSchema->type) && pToken->n == 0) {
return tscInvalidOperationMsg(msg, "invalid numeric data", pToken->z);
}
switch (pSchema->type) {
case TSDB_DATA_TYPE_BOOL: { // bool
if (isNullStr(pToken)) {
tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
} else {
if ((pToken->type == TK_BOOL || pToken->type == TK_STRING) && (pToken->n != 0)) {
if (strncmp(pToken->z, "true", pToken->n) == 0) {
tdAppendMemRowColVal(row, &TRUE_VALUE, true, colId, pSchema->type, toffset);
} else if (strncmp(pToken->z, "false", pToken->n) == 0) {
tdAppendMemRowColVal(row, &FALSE_VALUE, true, colId, pSchema->type, toffset);
} else {
return tscSQLSyntaxErrMsg(msg, "invalid bool data", pToken->z);
}
} else if (pToken->type == TK_INTEGER) {
iv = strtoll(pToken->z, NULL, 10);
tdAppendMemRowColVal(row, ((iv == 0) ? &FALSE_VALUE : &TRUE_VALUE), true, colId, pSchema->type, toffset);
} else if (pToken->type == TK_FLOAT) {
double dv = strtod(pToken->z, NULL);
tdAppendMemRowColVal(row, ((dv == 0) ? &FALSE_VALUE : &TRUE_VALUE), true, colId, pSchema->type, toffset);
} else {
return tscInvalidOperationMsg(msg, "invalid bool data", pToken->z);
}
}
break;
}
case TSDB_DATA_TYPE_TINYINT:
if (isNullStr(pToken)) {
tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
} else {
ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true);
if (ret != TSDB_CODE_SUCCESS) {
return tscInvalidOperationMsg(msg, "invalid tinyint data", pToken->z);
} else if (!IS_VALID_TINYINT(iv)) {
return tscInvalidOperationMsg(msg, "data overflow", pToken->z);
}
uint8_t tmpVal = (uint8_t)iv;
tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset);
}
break;
case TSDB_DATA_TYPE_UTINYINT:
if (isNullStr(pToken)) {
tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
} else {
ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false);
if (ret != TSDB_CODE_SUCCESS) {
return tscInvalidOperationMsg(msg, "invalid unsigned tinyint data", pToken->z);
} else if (!IS_VALID_UTINYINT(iv)) {
return tscInvalidOperationMsg(msg, "unsigned tinyint data overflow", pToken->z);
}
uint8_t tmpVal = (uint8_t)iv;
tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset);
}
break;
case TSDB_DATA_TYPE_SMALLINT:
if (isNullStr(pToken)) {
tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
} else {
ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true);
if (ret != TSDB_CODE_SUCCESS) {
return tscInvalidOperationMsg(msg, "invalid smallint data", pToken->z);
} else if (!IS_VALID_SMALLINT(iv)) {
return tscInvalidOperationMsg(msg, "smallint data overflow", pToken->z);
}
int16_t tmpVal = (int16_t)iv;
tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset);
}
break;
case TSDB_DATA_TYPE_USMALLINT:
if (isNullStr(pToken)) {
tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
} else {
ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false);
if (ret != TSDB_CODE_SUCCESS) {
return tscInvalidOperationMsg(msg, "invalid unsigned smallint data", pToken->z);
} else if (!IS_VALID_USMALLINT(iv)) {
return tscInvalidOperationMsg(msg, "unsigned smallint data overflow", pToken->z);
}
uint16_t tmpVal = (uint16_t)iv;
tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset);
}
break;
case TSDB_DATA_TYPE_INT:
if (isNullStr(pToken)) {
tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
} else {
ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true);
if (ret != TSDB_CODE_SUCCESS) {
return tscInvalidOperationMsg(msg, "invalid int data", pToken->z);
} else if (!IS_VALID_INT(iv)) {
return tscInvalidOperationMsg(msg, "int data overflow", pToken->z);
}
int32_t tmpVal = (int32_t)iv;
tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset);
}
break;
case TSDB_DATA_TYPE_UINT:
if (isNullStr(pToken)) {
tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
} else {
ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false);
if (ret != TSDB_CODE_SUCCESS) {
return tscInvalidOperationMsg(msg, "invalid unsigned int data", pToken->z);
} else if (!IS_VALID_UINT(iv)) {
return tscInvalidOperationMsg(msg, "unsigned int data overflow", pToken->z);
}
uint32_t tmpVal = (uint32_t)iv;
tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset);
}
break;
case TSDB_DATA_TYPE_BIGINT:
if (isNullStr(pToken)) {
tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
} else {
ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true);
if (ret != TSDB_CODE_SUCCESS) {
return tscInvalidOperationMsg(msg, "invalid bigint data", pToken->z);
} else if (!IS_VALID_BIGINT(iv)) {
return tscInvalidOperationMsg(msg, "bigint data overflow", pToken->z);
}
tdAppendMemRowColVal(row, &iv, true, colId, pSchema->type, toffset);
}
break;
case TSDB_DATA_TYPE_UBIGINT:
if (isNullStr(pToken)) {
tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
} else {
ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false);
if (ret != TSDB_CODE_SUCCESS) {
return tscInvalidOperationMsg(msg, "invalid unsigned bigint data", pToken->z);
} else if (!IS_VALID_UBIGINT((uint64_t)iv)) {
return tscInvalidOperationMsg(msg, "unsigned bigint data overflow", pToken->z);
}
uint64_t tmpVal = (uint64_t)iv;
tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset);
}
break;
case TSDB_DATA_TYPE_FLOAT:
if (isNullStr(pToken)) {
tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
} else {
double dv;
if (TK_ILLEGAL == tscToDouble(pToken, &dv, &endptr)) {
return tscInvalidOperationMsg(msg, "illegal float data", pToken->z);
}
if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) ||
isnan(dv)) {
return tscInvalidOperationMsg(msg, "illegal float data", pToken->z);
}
float tmpVal = (float)dv;
tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset);
}
break;
case TSDB_DATA_TYPE_DOUBLE:
if (isNullStr(pToken)) {
tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
} else {
double dv;
if (TK_ILLEGAL == tscToDouble(pToken, &dv, &endptr)) {
return tscInvalidOperationMsg(msg, "illegal double data", pToken->z);
}
if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) {
return tscInvalidOperationMsg(msg, "illegal double data", pToken->z);
}
tdAppendMemRowColVal(row, &dv, true, colId, pSchema->type, toffset);
}
break;
case TSDB_DATA_TYPE_BINARY:
// binary data cannot be null-terminated char string, otherwise the last char of the string is lost
if (pToken->type == TK_NULL) {
tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
} else { // too long values will return invalid sql, not be truncated automatically
if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { // todo refactor
return tscInvalidOperationMsg(msg, "string data overflow", pToken->z);
}
// STR_WITH_SIZE_TO_VARSTR(payload, pToken->z, pToken->n);
char *rowEnd = memRowEnd(row);
STR_WITH_SIZE_TO_VARSTR(rowEnd, pToken->z, pToken->n);
tdAppendMemRowColVal(row, rowEnd, false, colId, pSchema->type, toffset);
}
break;
case TSDB_DATA_TYPE_NCHAR:
if (pToken->type == TK_NULL) {
tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
} else {
// if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long'
int32_t output = 0;
char * rowEnd = memRowEnd(row);
if (!taosMbsToUcs4(pToken->z, pToken->n, (char *)varDataVal(rowEnd), pSchema->bytes - VARSTR_HEADER_SIZE,
&output)) {
char buf[512] = {0};
snprintf(buf, tListLen(buf), "%s", strerror(errno));
return tscInvalidOperationMsg(msg, buf, pToken->z);
}
varDataSetLen(rowEnd, output);
tdAppendMemRowColVal(row, rowEnd, false, colId, pSchema->type, toffset);
}
break;
case TSDB_DATA_TYPE_TIMESTAMP: {
if (pToken->type == TK_NULL) {
if (primaryKey) {
// When building SKVRow primaryKey, we should not skip even with NULL value.
int64_t tmpVal = 0;
tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset);
} else {
tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
}
} else {
int64_t tmpVal;
if (tsParseTime(pToken, &tmpVal, str, msg, timePrec) != TSDB_CODE_SUCCESS) {
return tscInvalidOperationMsg(msg, "invalid timestamp", pToken->z);
}
tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset);
}
break;
}
}
return TSDB_CODE_SUCCESS;
}
#ifdef __cplusplus
}
#endif
#endif

237
2.0/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h
View File

@ -1,237 +0,0 @@
/* DO NOT EDIT THIS FILE - it is machine generated */
#include <jni.h>
/* Header for class com_taosdata_jdbc_TSDBJNIConnector */
#ifndef _Included_com_taosdata_jdbc_TSDBJNIConnector
#define _Included_com_taosdata_jdbc_TSDBJNIConnector
#ifdef __cplusplus
extern "C" {
#endif
#undef com_taosdata_jdbc_TSDBJNIConnector_INVALID_CONNECTION_POINTER_VALUE
#define com_taosdata_jdbc_TSDBJNIConnector_INVALID_CONNECTION_POINTER_VALUE 0LL
/*
* Class: com_taosdata_jdbc_TSDBJNIConnector
* Method:
* Signature: (Ljava/lang/String;)V
*/
JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setAllocModeImp
(JNIEnv *, jclass, jint, jstring, jboolean);
/*
* Class: com_taosdata_jdbc_TSDBJNIConnector
* Method:
* Signature: ()Ljava/lang/String;
*/
JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_dumpMemoryLeakImp
(JNIEnv *, jclass);
/*
* Class: com_taosdata_jdbc_TSDBJNIConnector
* Method: initImp
* Signature: (Ljava/lang/String;)V
*/
JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_initImp
(JNIEnv *, jclass, jstring);
/*
* Class: com_taosdata_jdbc_TSDBJNIConnector
* Method: setOptions
* Signature: (ILjava/lang/String;)I
*/
JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setOptions
(JNIEnv *, jclass, jint, jstring);
/*
* Class: com_taosdata_jdbc_TSDBJNIConnector
* Method: getTsCharset
* Signature: ()Ljava/lang/String;
*/
JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getTsCharset
(JNIEnv *, jclass);
/*
* Class: com_taosdata_jdbc_TSDBJNIConnector
* Method: getResultTimePrecisionImp
* Signature: (JJ)I
*/
JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TDDBJNIConnector_getResultTimePrecisionImp
(JNIEnv *, jobject, jlong, jlong);
/*
* Class: com_taosdata_jdbc_TSDBJNIConnector
* Method: connectImp
* Signature: (Ljava/lang/String;ILjava/lang/String;Ljava/lang/String;Ljava/lang/String;)J
*/
JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_connectImp
(JNIEnv *, jobject, jstring, jint, jstring, jstring, jstring);
/*
* Class: com_taosdata_jdbc_TSDBJNIConnector
* Method: executeQueryImp
* Signature: ([BJ)I
*/
JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeQueryImp
(JNIEnv *, jobject, jbyteArray, jlong);
/*
* Class: com_taosdata_jdbc_TSDBJNIConnector
* Method: getErrCodeImp
* Signature: (J)I
*/
JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getErrCodeImp
(JNIEnv *, jobject, jlong, jlong);
/*
* Class: com_taosdata_jdbc_TSDBJNIConnector
* Method: getErrMsgImp
* Signature: (J)Ljava/lang/String;
*/
JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getErrMsgImp
(JNIEnv *, jobject, jlong);
/*
* Class: com_taosdata_jdbc_TSDBJNIConnector
* Method: getResultSetImp
* Signature: (J)J
*/
JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getResultSetImp
(JNIEnv *env, jobject jobj, jlong con, jlong tres);
/*
* Class: com_taosdata_jdbc_TSDBJNIConnector
* Method: isUpdateQueryImp
* Signature: (JJ)I
*/
JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_isUpdateQueryImp
(JNIEnv *env, jobject jobj, jlong con, jlong tres);
/*
* Class: com_taosdata_jdbc_TSDBJNIConnector
* Method: freeResultSetImp
* Signature: (JJ)I
*/
JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_freeResultSetImp
(JNIEnv *, jobject, jlong, jlong);
/*
* Class: com_taosdata_jdbc_TSDBJNIConnector
* Method: getAffectedRowsImp
* Signature: (J)I
*/
JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getAffectedRowsImp
(JNIEnv *env, jobject jobj, jlong con, jlong res);
/*
* Class: com_taosdata_jdbc_TSDBJNIConnector
* Method: getSchemaMetaDataImp
* Signature: (JJLjava/util/List;)I
*/
JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getSchemaMetaDataImp
(JNIEnv *, jobject, jlong, jlong, jobject);
/*
* Class: com_taosdata_jdbc_TSDBJNIConnector
* Method: fetchRowImp
* Signature: (JJLcom/taosdata/jdbc/TSDBResultSetRowData;)I
*/
JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchRowImp
(JNIEnv *, jobject, jlong, jlong, jobject);
/*
* Class: com_taosdata_jdbc_TSDBJNIConnector
* Method: fetchBlockImp
* Signature: (JJLcom/taosdata/jdbc/TSDBResultSetBlockData;)I
*/
JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchBlockImp
(JNIEnv *, jobject, jlong, jlong, jobject);
/*
* Class: com_taosdata_jdbc_TSDBJNIConnector
* Method: closeConnectionImp
* Signature: (J)I
*/
JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_closeConnectionImp
(JNIEnv *, jobject, jlong);
/*
* Class: com_taosdata_jdbc_TSDBJNIConnector
* Method: subscribeImp
* Signature: (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;JI)J
*/
JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_subscribeImp
(JNIEnv *, jobject, jlong, jboolean, jstring, jstring, jint);
/*
* Class: com_taosdata_jdbc_TSDBJNIConnector
* Method: consumeImp
* Signature: (J)Lcom/taosdata/jdbc/TSDBResultSetRowData;
*/
JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_consumeImp
(JNIEnv *, jobject, jlong);
/*
* Class: com_taosdata_jdbc_TSDBJNIConnector
* Method: unsubscribeImp
* Signature: (J)V
*/
JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_unsubscribeImp
(JNIEnv *, jobject, jlong, jboolean);
/*
* Class: com_taosdata_jdbc_TSDBJNIConnector
* Method: validateCreateTableSqlImp
* Signature: (J[B)I
*/
JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_validateCreateTableSqlImp
(JNIEnv *, jobject, jlong, jbyteArray);
/*
* Class: com_taosdata_jdbc_TSDBJNIConnector
* Method: prepareStmtImp
* Signature: ([BJ)I
*/
JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_prepareStmtImp
(JNIEnv *, jobject, jbyteArray, jlong);
/*
* Class: com_taosdata_jdbc_TSDBJNIConnector
* Method: setBindTableNameImp
* Signature: (JLjava/lang/String;J)I
*/
JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setBindTableNameImp
(JNIEnv *, jobject, jlong, jstring, jlong);
/*
* Class: com_taosdata_jdbc_TSDBJNIConnector
* Method: bindColDataImp
* Signature: (J[B[B[BIIIIJ)J
*/
JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_bindColDataImp
(JNIEnv *, jobject, jlong, jbyteArray, jbyteArray, jbyteArray, jint, jint, jint, jint, jlong);
/*
* Class: com_taosdata_jdbc_TSDBJNIConnector
* Method: executeBatchImp
* Signature: (JJ)I
*/
JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeBatchImp(JNIEnv *env, jobject jobj, jlong stmt, jlong con);
/*
* Class: com_taosdata_jdbc_TSDBJNIConnector
* Method: closeStmt
* Signature: (JJ)I
*/
JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_closeStmt(JNIEnv *env, jobject jobj, jlong stmt, jlong con);
/**
* Class: com_taosdata_jdbc_TSDBJNIConnector
* Method: setTableNameTagsImp
* Signature: (JLjava/lang/String;I[B[B[B[BJ)I
*/
JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setTableNameTagsImp
(JNIEnv *, jobject, jlong, jstring, jint, jbyteArray, jbyteArray, jbyteArray, jbyteArray, jlong);
#ifdef __cplusplus
}
#endif
#endif

1029
2.0/src/client/src/TSDBJNIConnector.c
View File

File diff suppressed because it is too large Load Diff

45
2.0/src/client/src/taos.def
View File

@ -1,45 +0,0 @@
EXPORTS
taos_init
taos_cleanup
taos_options
taos_connect
taos_connect_auth
taos_close
taos_stmt_init
taos_stmt_prepare
taos_stmt_set_tbname_tags
taos_stmt_set_tbname
taos_stmt_is_insert
taos_stmt_num_params
taos_stmt_bind_param
taos_stmt_add_batch
taos_stmt_execute
taos_stmt_use_result
taos_stmt_close
taos_stmt_errstr
taos_query
taos_fetch_row
taos_result_precision
taos_free_result
taos_field_count
taos_num_fields
taos_affected_rows
taos_fetch_fields
taos_select_db
taos_print_row
taos_stop_query
taos_fetch_block
taos_validate_sql
taos_fetch_lengths
taos_get_server_info
taos_get_client_info
taos_errstr
taos_errno
taos_query_a
taos_fetch_rows_a
taos_subscribe
taos_consume
taos_unsubscribe
taos_open_stream
taos_close_stream
taos_load_table_info

31
2.0/src/client/src/taos.rc.in
View File

@ -1,31 +0,0 @@
1 VERSIONINFO
FILEVERSION ${TD_VER_NUMBER}
PRODUCTVERSION ${TD_VER_NUMBER}
FILEFLAGSMASK 0x17L
#ifdef _DEBUG
FILEFLAGS 0x1L
#else
FILEFLAGS 0x0L
#endif
FILEOS 0x4L
FILETYPE 0x0L
FILESUBTYPE 0x0L
BEGIN
BLOCK "StringFileInfo"
BEGIN
BLOCK "040904b0"
BEGIN
VALUE "FileDescription", "Native C Driver for TDengine"
VALUE "FileVersion", "${TD_VER_NUMBER}"
VALUE "InternalName", "taos.dll(${TD_VER_CPUTYPE})"
VALUE "LegalCopyright", "Copyright (C) 2020 TAOS Data"
VALUE "OriginalFilename", ""
VALUE "ProductName", "taos.dll(${TD_VER_CPUTYPE})"
VALUE "ProductVersion", "${TD_VER_NUMBER}"
END
END
BLOCK "VarFileInfo"
BEGIN
VALUE "Translation", 0x409, 1200
END
END

419
2.0/src/client/src/tscAsync.c
View File

@ -1,419 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "tutil.h"
#include "tnote.h"
#include "trpc.h"
#include "tscLog.h"
#include "tscSubquery.h"
#include "tscUtil.h"
#include "tsched.h"
#include "qTableMeta.h"
#include "tsclient.h"
static void tscAsyncQueryRowsForNextVnode(void *param, TAOS_RES *tres, int numOfRows);
/*
* Proxy function to perform sequentially query&retrieve operation.
* If sql queries upon a super table and two-stage merge procedure is not involved (when employ the projection
* query), it will sequentially query&retrieve data for all vnodes
*/
static void tscAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows);
void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, __async_cb_func_t fp, void* param, const char* sqlstr, size_t sqlLen) {
SSqlCmd* pCmd = &pSql->cmd;
pSql->signature = pSql;
pSql->param = param;
pSql->pTscObj = pObj;
pSql->parseRetry= 0;
pSql->maxRetry = TSDB_MAX_REPLICA;
pSql->fp = fp;
pSql->fetchFp = fp;
registerSqlObj(pSql);
pSql->sqlstr = calloc(1, sqlLen + 1);
if (pSql->sqlstr == NULL) {
tscError("0x%"PRIx64" failed to malloc sql string buffer", pSql->self);
pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY;
tscAsyncResultOnError(pSql);
return;
}
strntolower(pSql->sqlstr, sqlstr, (int32_t)sqlLen);
tscDebugL("0x%"PRIx64" SQL: %s", pSql->self, pSql->sqlstr);
pCmd->resColumnId = TSDB_RES_COL_ID;
taosAcquireRef(tscObjRef, pSql->self);
int32_t code = tsParseSql(pSql, true);
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
taosReleaseRef(tscObjRef, pSql->self);
return;
}
if (code != TSDB_CODE_SUCCESS) {
pSql->res.code = code;
tscAsyncResultOnError(pSql);
taosReleaseRef(tscObjRef, pSql->self);
return;
}
SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd);
executeQuery(pSql, pQueryInfo);
taosReleaseRef(tscObjRef, pSql->self);
}
// TODO return the correct error code to client in tscQueueAsyncError
void taos_query_a(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *param) {
taos_query_ra(taos, sqlstr, fp, param);
}
TAOS_RES * taos_query_ra(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *param) {
STscObj *pObj = (STscObj *)taos;
if (pObj == NULL || pObj->signature != pObj) {
tscError("pObj:%p is NULL or freed", pObj);
terrno = TSDB_CODE_TSC_DISCONNECTED;
tscQueueAsyncError(fp, param, TSDB_CODE_TSC_DISCONNECTED);
return NULL;
}
int32_t sqlLen = (int32_t)strlen(sqlstr);
if (sqlLen > tsMaxSQLStringLen) {
tscError("sql string exceeds max length:%d", tsMaxSQLStringLen);
terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT;
tscQueueAsyncError(fp, param, terrno);
return NULL;
}
nPrintTsc("%s", sqlstr);
SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj));
if (pSql == NULL) {
tscError("failed to malloc sqlObj");
tscQueueAsyncError(fp, param, TSDB_CODE_TSC_OUT_OF_MEMORY);
return NULL;
}
doAsyncQuery(pObj, pSql, fp, param, sqlstr, sqlLen);
return pSql;
}
static void tscAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows) {
if (tres == NULL) {
return;
}
SSqlObj *pSql = (SSqlObj *)tres;
SSqlRes *pRes = &pSql->res;
SSqlCmd *pCmd = &pSql->cmd;
if (numOfRows == 0) {
if (hasMoreVnodesToTry(pSql)) { // sequentially retrieve data from remain vnodes.
tscTryQueryNextVnode(pSql, tscAsyncQueryRowsForNextVnode);
} else {
/*
* all available virtual node has been checked already, now we need to check
* for the next subclause queries
*/
if (pCmd->active->sibling != NULL) {
pCmd->active = pCmd->active->sibling;
tscTryQueryNextClause(pSql, tscAsyncQueryRowsForNextVnode);
return;
}
/*
* 1. has reach the limitation
* 2. no remain virtual nodes to be retrieved anymore
*/
(*pSql->fetchFp)(param, pSql, 0);
}
return;
}
// local merge has handle this situation during super table non-projection query.
if (pCmd->command != TSDB_SQL_RETRIEVE_GLOBALMERGE) {
pRes->numOfClauseTotal += pRes->numOfRows;
}
(*pSql->fetchFp)(param, tres, numOfRows);
}
// actual continue retrieve function with user-specified callback function
static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRows, __async_cb_func_t fp) {
SSqlObj *pSql = (SSqlObj *)tres;
if (pSql == NULL) { // error
tscError("sql object is NULL");
return;
}
SSqlCmd *pCmd = &pSql->cmd;
SSqlRes *pRes = &pSql->res;
if ((pRes->qId == 0 || numOfRows != 0) && pCmd->command < TSDB_SQL_LOCAL) {
if (pRes->qId == 0 && numOfRows != 0) {
tscError("qhandle is NULL");
} else {
pRes->code = numOfRows;
}
tscAsyncResultOnError(pSql);
return;
}
pSql->fp = fp;
if (pCmd->command != TSDB_SQL_RETRIEVE_GLOBALMERGE && pCmd->command < TSDB_SQL_LOCAL) {
pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE_MNODE : TSDB_SQL_FETCH;
}
if (pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) {
tscFetchDatablockForSubquery(pSql);
} else {
tscBuildAndSendRequest(pSql, NULL);
}
}
/*
* retrieve callback for fetch rows proxy.
* The below two functions both serve as the callback function of query virtual node.
* query callback first, and then followed by retrieve callback
*/
static void tscAsyncQueryRowsForNextVnode(void *param, TAOS_RES *tres, int numOfRows) {
// query completed, continue to retrieve
tscProcessAsyncRetrieveImpl(param, tres, numOfRows, tscAsyncFetchRowsProxy);
}
void taos_fetch_rows_a(TAOS_RES *tres, __async_cb_func_t fp, void *param) {
SSqlObj *pSql = (SSqlObj *)tres;
if (pSql == NULL || pSql->signature != pSql) {
tscError("sql object is NULL");
tscQueueAsyncError(fp, param, TSDB_CODE_TSC_DISCONNECTED);
return;
}
SSqlRes *pRes = &pSql->res;
SSqlCmd *pCmd = &pSql->cmd;
// user-defined callback function is stored in fetchFp
pSql->fetchFp = fp;
pSql->fp = tscAsyncFetchRowsProxy;
pSql->param = param;
tscResetForNextRetrieve(pRes);
// handle outer query based on the already retrieved nest query results.
SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd);
if (pQueryInfo->pUpstream != NULL && taosArrayGetSize(pQueryInfo->pUpstream) > 0) {
SSchedMsg schedMsg = {0};
schedMsg.fp = doRetrieveSubqueryData;
schedMsg.ahandle = (void *)pSql;
schedMsg.thandle = (void *)1;
schedMsg.msg = 0;
taosScheduleTask(tscQhandle, &schedMsg);
return;
}
if (pRes->qId == 0) {
tscError("qhandle is invalid");
pRes->code = TSDB_CODE_TSC_INVALID_QHANDLE;
tscAsyncResultOnError(pSql);
return;
}
if (pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) {
tscFetchDatablockForSubquery(pSql);
} else if (pRes->completed) {
if(pCmd->command == TSDB_SQL_FETCH || (pCmd->command >= TSDB_SQL_SERV_STATUS && pCmd->command <= TSDB_SQL_CURRENT_USER)) {
if (hasMoreVnodesToTry(pSql)) { // sequentially retrieve data from remain vnodes.
tscTryQueryNextVnode(pSql, tscAsyncQueryRowsForNextVnode);
} else {
/*
* all available virtual nodes in current clause has been checked already, now try the
* next one in the following union subclause
*/
if (pCmd->active->sibling != NULL) {
pCmd->active = pCmd->active->sibling; // todo refactor
tscTryQueryNextClause(pSql, tscAsyncQueryRowsForNextVnode);
return;
}
/*
* 1. has reach the limitation
* 2. no remain virtual nodes to be retrieved anymore
*/
(*pSql->fetchFp)(param, pSql, 0);
}
return;
} else if (pCmd->command == TSDB_SQL_RETRIEVE_MNODE || pCmd->command == TSDB_SQL_RETRIEVE_GLOBALMERGE) {
// in case of show command, return no data
(*pSql->fetchFp)(param, pSql, 0);
} else {
assert(0);
}
} else { // current query is not completed, continue retrieve from node
if (pCmd->command != TSDB_SQL_RETRIEVE_GLOBALMERGE && pCmd->command < TSDB_SQL_LOCAL) {
pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE_MNODE : TSDB_SQL_FETCH;
}
SQueryInfo* pQueryInfo1 = tscGetQueryInfo(&pSql->cmd);
tscBuildAndSendRequest(pSql, pQueryInfo1);
}
}
// this function will be executed by queue task threads, so the terrno is not valid
static void tscProcessAsyncError(SSchedMsg *pMsg) {
void (*fp)() = pMsg->ahandle;
terrno = *(int32_t*) pMsg->msg;
tfree(pMsg->msg);
(*fp)(pMsg->thandle, NULL, terrno);
}
void tscQueueAsyncError(void(*fp), void *param, int32_t code) {
int32_t* c = malloc(sizeof(int32_t));
*c = code;
SSchedMsg schedMsg = {0};
schedMsg.fp = tscProcessAsyncError;
schedMsg.ahandle = fp;
schedMsg.thandle = param;
schedMsg.msg = c;
taosScheduleTask(tscQhandle, &schedMsg);
}
static void tscAsyncResultCallback(SSchedMsg *pMsg) {
SSqlObj* pSql = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)pMsg->ahandle);
if (pSql == NULL || pSql->signature != pSql) {
tscDebug("%p SqlObj is freed, not add into queue async res", pMsg->ahandle);
return;
}
assert(pSql->res.code != TSDB_CODE_SUCCESS);
tscError("0x%"PRIx64" async result callback, code:%s", pSql->self, tstrerror(pSql->res.code));
SSqlRes *pRes = &pSql->res;
if (pSql->fp == NULL || pSql->fetchFp == NULL){
taosReleaseRef(tscObjRef, pSql->self);
return;
}
pSql->fp = pSql->fetchFp;
(*pSql->fp)(pSql->param, pSql, pRes->code);
taosReleaseRef(tscObjRef, pSql->self);
}
void tscAsyncResultOnError(SSqlObj* pSql) {
SSchedMsg schedMsg = {0};
schedMsg.fp = tscAsyncResultCallback;
schedMsg.ahandle = (void *)pSql->self;
schedMsg.thandle = (void *)1;
schedMsg.msg = 0;
taosScheduleTask(tscQhandle, &schedMsg);
}
int tscSendMsgToServer(SSqlObj *pSql);
void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
SSqlObj* pSql = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)param);
if (pSql == NULL) return;
assert(pSql->signature == pSql && (int64_t)param == pSql->self);
SSqlCmd *pCmd = &pSql->cmd;
SSqlRes *pRes = &pSql->res;
pRes->code = code;
SSqlObj *sub = (SSqlObj*) res;
const char* msg = (sub->cmd.command == TSDB_SQL_STABLEVGROUP)? "vgroup-list":"multi-tableMeta";
if (code != TSDB_CODE_SUCCESS) {
tscError("0x%"PRIx64" get %s failed, code:%s", pSql->self, msg, tstrerror(code));
if (code == TSDB_CODE_RPC_FQDN_ERROR) {
size_t sz = strlen(tscGetErrorMsgPayload(&sub->cmd));
tscAllocPayload(&pSql->cmd, (int)sz + 1);
memcpy(tscGetErrorMsgPayload(&pSql->cmd), tscGetErrorMsgPayload(&sub->cmd), sz);
}
goto _error;
}
tscDebug("0x%"PRIx64" get %s successfully", pSql->self, msg);
if (pSql->pStream == NULL) {
SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd);
if (pQueryInfo != NULL && TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_INSERT)) {
tscDebug("0x%" PRIx64 " continue parse sql after get table-meta", pSql->self);
code = tsParseSql(pSql, false);
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
taosReleaseRef(tscObjRef, pSql->self);
return;
} else if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
if (TSDB_QUERY_HAS_TYPE(pCmd->insertParam.insertType, TSDB_QUERY_TYPE_STMT_INSERT)) { // stmt insert
(*pSql->fp)(pSql->param, pSql, code);
} else if (TSDB_QUERY_HAS_TYPE(pCmd->insertParam.insertType, TSDB_QUERY_TYPE_FILE_INSERT)) { // file insert
tscImportDataFromFile(pSql);
} else { // sql string insert
tscHandleMultivnodeInsert(pSql);
}
} else {
if (pSql->retryReason != TSDB_CODE_SUCCESS) {
tscDebug("0x%" PRIx64 " update cached table-meta, re-validate sql statement and send query again", pSql->self);
pSql->retryReason = TSDB_CODE_SUCCESS;
} else {
tscDebug("0x%" PRIx64 " cached table-meta, continue validate sql statement and send query", pSql->self);
}
code = tsParseSql(pSql, true);
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
taosReleaseRef(tscObjRef, pSql->self);
return;
} else if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
SQueryInfo *pQueryInfo1 = tscGetQueryInfo(pCmd);
executeQuery(pSql, pQueryInfo1);
}
taosReleaseRef(tscObjRef, pSql->self);
return;
} else { // stream computing
tscDebug("0x%"PRIx64" stream:%p meta is updated, start new query, command:%d", pSql->self, pSql->pStream, pCmd->command);
SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd);
if (tscNumOfExprs(pQueryInfo) == 0) {
tsParseSql(pSql, false);
}
(*pSql->fp)(pSql->param, pSql, code);
taosReleaseRef(tscObjRef, pSql->self);
return;
}
_error:
pRes->code = code;
tscAsyncResultOnError(pSql);
taosReleaseRef(tscObjRef, pSql->self);
}

1144
2.0/src/client/src/tscGlobalmerge.c
View File

File diff suppressed because it is too large Load Diff

955
2.0/src/client/src/tscLocal.c
View File

@ -1,955 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "tmsg.h"
#include "../../../include/client/taos.h"
#include "qTableMeta.h"
#include "taosdef.h"
#include "tname.h"
#include "tscLog.h"
#include "tscSubquery.h"
#include "tscUtil.h"
#include "tsclient.h"
#define STR_NOCASE_EQUAL(str1, len1, str2, len2) ((len1 == len2) && 0 == strncasecmp(str1, str2, len1))
typedef enum BuildType {
SCREATE_BUILD_TABLE = 1,
SCREATE_BUILD_DB = 2,
} BuildType;
typedef enum Stage {
SCREATE_CALLBACK_QUERY = 1,
SCREATE_CALLBACK_RETRIEVE = 2,
} Stage;
// support 'show create table'
typedef struct SCreateBuilder {
char sTableName[TSDB_TABLE_FNAME_LEN];
char buf[TSDB_TABLE_FNAME_LEN];
SSqlObj *pParentSql;
SSqlObj *pInterSql;
int32_t (*fp)(void *para, char* result);
Stage callStage;
} SCreateBuilder;
static void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnName, int16_t type, size_t valueLength);
static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
SSqlRes *pRes = &pSql->res;
// one column for each row
SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd);
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
STableMeta * pMeta = pTableMetaInfo->pTableMeta;
/*
* tagValueCnt is to denote the number of tags columns for meter, not metric. and is to show the column data.
* for meter, which is created according to metric, the value of tagValueCnt is not 0, and the numOfTags must be 0.
* for metric, the value of tagValueCnt must be 0, but the numOfTags is not 0
*/
int32_t numOfRows = tscGetNumOfColumns(pMeta);
int32_t totalNumOfRows = numOfRows + tscGetNumOfTags(pMeta);
if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
numOfRows = numOfRows + tscGetNumOfTags(pMeta);
}
pSql->res.pMerger = tscInitResObjForLocalQuery(totalNumOfRows, rowLen, pSql->self);
tscInitResForMerge(&pSql->res);
SSchema *pSchema = tscGetTableSchema(pMeta);
for (int32_t i = 0; i < numOfRows; ++i) {
TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0);
char* dst = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 0) * totalNumOfRows + pField->bytes * i;
STR_WITH_MAXSIZE_TO_VARSTR(dst, pSchema[i].name, pField->bytes);
char *type = tDataTypes[pSchema[i].type].name;
pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 1);
dst = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 1) * totalNumOfRows + pField->bytes * i;
STR_WITH_MAXSIZE_TO_VARSTR(dst, type, pField->bytes);
int32_t bytes = pSchema[i].bytes;
if (pSchema[i].type == TSDB_DATA_TYPE_BINARY || pSchema[i].type == TSDB_DATA_TYPE_NCHAR) {
bytes -= VARSTR_HEADER_SIZE;
if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR) {
bytes = bytes / TSDB_NCHAR_SIZE;
}
}
pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 2);
*(int32_t *)(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 2) * totalNumOfRows + pField->bytes * i) = bytes;
pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 3);
if (i >= tscGetNumOfColumns(pMeta) && tscGetNumOfTags(pMeta) != 0) {
char* output = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 3) * totalNumOfRows + pField->bytes * i;
const char *src = "TAG";
STR_WITH_MAXSIZE_TO_VARSTR(output, src, pField->bytes);
}
}
if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
return 0;
}
// the following is handle display tags for table created according to super table
for (int32_t i = numOfRows; i < totalNumOfRows; ++i) {
// field name
TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0);
char* output = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 0) * totalNumOfRows + pField->bytes * i;
STR_WITH_MAXSIZE_TO_VARSTR(output, pSchema[i].name, pField->bytes);
// type name
pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 1);
char *type = tDataTypes[pSchema[i].type].name;
output = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 1) * totalNumOfRows + pField->bytes * i;
STR_WITH_MAXSIZE_TO_VARSTR(output, type, pField->bytes);
// type length
int32_t bytes = pSchema[i].bytes;
pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 2);
if (pSchema[i].type == TSDB_DATA_TYPE_BINARY || pSchema[i].type == TSDB_DATA_TYPE_NCHAR) {
bytes -= VARSTR_HEADER_SIZE;
if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR) {
bytes = bytes / TSDB_NCHAR_SIZE;
}
}
*(int32_t *)(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 2) * totalNumOfRows + pField->bytes * i) = bytes;
// tag value
pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 3);
char *target = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 3) * totalNumOfRows + pField->bytes * i;
const char *src = "TAG";
STR_WITH_MAXSIZE_TO_VARSTR(target, src, pField->bytes);
}
return 0;
}
static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols, int32_t typeColLength,
int32_t noteColLength) {
int32_t rowLen = 0;
SColumnIndex index = {0};
pSql->cmd.numOfCols = numOfCols;
SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd);
pQueryInfo->order.order = TSDB_ORDER_ASC;
TAOS_FIELD f = {.type = TSDB_DATA_TYPE_BINARY, .bytes = (TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE};
tstrncpy(f.name, "Field", sizeof(f.name));
SInternalField* pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY,
(TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE, -1000, (TSDB_COL_NAME_LEN - 1), false);
rowLen += ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE);
f.bytes = (int16_t)(typeColLength + VARSTR_HEADER_SIZE);
f.type = TSDB_DATA_TYPE_BINARY;
tstrncpy(f.name, "Type", sizeof(f.name));
pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, (int16_t)(typeColLength + VARSTR_HEADER_SIZE),
-1000, typeColLength, false);
rowLen += typeColLength + VARSTR_HEADER_SIZE;
f.bytes = sizeof(int32_t);
f.type = TSDB_DATA_TYPE_INT;
tstrncpy(f.name, "Length", sizeof(f.name));
pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_INT, sizeof(int32_t),
-1000, sizeof(int32_t), false);
rowLen += sizeof(int32_t);
f.bytes = (int16_t)(noteColLength + VARSTR_HEADER_SIZE);
f.type = TSDB_DATA_TYPE_BINARY;
tstrncpy(f.name, "Note", sizeof(f.name));
pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, (int16_t)(noteColLength + VARSTR_HEADER_SIZE),
-1000, noteColLength, false);
rowLen += noteColLength + VARSTR_HEADER_SIZE;
return rowLen;
}
static int32_t tscProcessDescribeTable(SSqlObj *pSql) {
SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd);
assert(tscGetMetaInfo(pQueryInfo, 0)->pTableMeta != NULL);
const int32_t NUM_OF_DESC_TABLE_COLUMNS = 4;
const int32_t TYPE_COLUMN_LENGTH = 20;
const int32_t NOTE_COLUMN_MIN_LENGTH = 8;
int32_t noteFieldLen = NOTE_COLUMN_MIN_LENGTH;
int32_t rowLen = tscBuildTableSchemaResultFields(pSql, NUM_OF_DESC_TABLE_COLUMNS, TYPE_COLUMN_LENGTH, noteFieldLen);
tscFieldInfoUpdateOffset(pQueryInfo);
return tscSetValueToResObj(pSql, rowLen);
}
static int32_t tscGetNthFieldResult(TAOS_ROW row, TAOS_FIELD* fields, int *lengths, int idx, char *result) {
const char *val = (const char*)row[idx];
if (val == NULL) {
sprintf(result, "%s", TSDB_DATA_NULL_STR);
return -1;
}
uint8_t type = fields[idx].type;
int32_t length = lengths[idx];
switch (type) {
case TSDB_DATA_TYPE_BOOL:
sprintf(result, "%s", ((((int32_t)(*((char *)val))) == 1) ? "true" : "false"));
break;
case TSDB_DATA_TYPE_TINYINT:
sprintf(result, "%d", *((int8_t *)val));
break;
case TSDB_DATA_TYPE_SMALLINT:
sprintf(result, "%d", *((int16_t *)val));
break;
case TSDB_DATA_TYPE_INT:
sprintf(result, "%d", *((int32_t *)val));
break;
case TSDB_DATA_TYPE_BIGINT:
sprintf(result, "%"PRId64, *((int64_t *)val));
break;
case TSDB_DATA_TYPE_FLOAT:
sprintf(result, "%f", GET_FLOAT_VAL(val));
break;
case TSDB_DATA_TYPE_DOUBLE:
sprintf(result, "%f", GET_DOUBLE_VAL(val));
break;
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_BINARY:
memcpy(result, val, length);
break;
case TSDB_DATA_TYPE_TIMESTAMP:
///formatTimestamp(buf, *(int64_t*)val, TSDB_TIME_PRECISION_MICRO);
//memcpy(result, val, strlen(buf));
sprintf(result, "%"PRId64, *((int64_t *)val));
break;
default:
break;
}
return 0;
}
void tscSCreateCallBack(void *param, TAOS_RES *tres, int code) {
if (param == NULL || tres == NULL) {
return;
}
SCreateBuilder *builder = (SCreateBuilder *)(param);
SSqlObj *pParentSql = builder->pParentSql;
SSqlObj *pSql = (SSqlObj *)tres;
SSqlRes *pRes = &pParentSql->res;
pRes->code = taos_errno(pSql);
if (pRes->code != TSDB_CODE_SUCCESS) {
taos_free_result(pSql);
free(builder);
tscAsyncResultOnError(pParentSql);
return;
}
if (builder->callStage == SCREATE_CALLBACK_QUERY) {
taos_fetch_rows_a(tres, tscSCreateCallBack, param);
builder->callStage = SCREATE_CALLBACK_RETRIEVE;
} else {
char *result = calloc(1, TSDB_MAX_BINARY_LEN);
pRes->code = builder->fp(builder, result);
taos_free_result(pSql);
free(builder);
free(result);
if (pRes->code == TSDB_CODE_SUCCESS) {
(*pParentSql->fp)(pParentSql->param, pParentSql, code);
} else {
tscAsyncResultOnError(pParentSql);
}
}
}
TAOS_ROW tscFetchRow(void *param) {
SCreateBuilder *builder = (SCreateBuilder *)param;
if (builder == NULL) {
return NULL;
}
SSqlObj *pSql = builder->pInterSql;
if (pSql == NULL || pSql->signature != pSql) {
terrno = TSDB_CODE_TSC_DISCONNECTED;
return NULL;
}
SSqlCmd *pCmd = &pSql->cmd;
SSqlRes *pRes = &pSql->res;
if (pRes->qId == 0 ||
pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT ||
pCmd->command == TSDB_SQL_INSERT) {
return NULL;
}
// set the sql object owner
tscSetSqlOwner(pSql);
// current data set are exhausted, fetch more data from node
if (pRes->row >= pRes->numOfRows && (pRes->completed != true || hasMoreVnodesToTry(pSql) || hasMoreClauseToTry(pSql)) &&
(pCmd->command == TSDB_SQL_RETRIEVE_MNODE ||
pCmd->command == TSDB_SQL_RETRIEVE_GLOBALMERGE ||
pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE ||
pCmd->command == TSDB_SQL_FETCH ||
pCmd->command == TSDB_SQL_SHOW ||
pCmd->command == TSDB_SQL_SHOW_CREATE_TABLE ||
pCmd->command == TSDB_SQL_SHOW_CREATE_STABLE ||
pCmd->command == TSDB_SQL_SHOW_CREATE_DATABASE ||
pCmd->command == TSDB_SQL_SELECT ||
pCmd->command == TSDB_SQL_DESCRIBE_TABLE ||
pCmd->command == TSDB_SQL_SERV_STATUS ||
pCmd->command == TSDB_SQL_CURRENT_DB ||
pCmd->command == TSDB_SQL_SERV_VERSION ||
pCmd->command == TSDB_SQL_CLI_VERSION ||
pCmd->command == TSDB_SQL_CURRENT_USER )) {
taos_fetch_rows_a(pSql, tscSCreateCallBack, param);
return NULL;
}
void* data = doSetResultRowData(pSql);
tscClearSqlOwner(pSql);
return data;
}
static int32_t tscGetTableTagValue(SCreateBuilder *builder, char *result) {
TAOS_ROW row = tscFetchRow(builder);
SSqlObj* pSql = builder->pInterSql;
if (row == NULL) {
return TSDB_CODE_TSC_INVALID_TABLE_NAME;
}
int32_t* lengths = taos_fetch_lengths(pSql);
int num_fields = taos_num_fields(pSql);
TAOS_FIELD *fields = taos_fetch_fields(pSql);
char buf[TSDB_COL_NAME_LEN + 16];
for (int i = 0; i < num_fields; i++) {
memset(buf, 0, sizeof(buf));
int32_t ret = tscGetNthFieldResult(row, fields, lengths, i, buf);
if (i == 0) {
snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s", "(");
}
if ((fields[i].type == TSDB_DATA_TYPE_NCHAR
|| fields[i].type == TSDB_DATA_TYPE_BINARY
|| fields[i].type == TSDB_DATA_TYPE_TIMESTAMP) && 0 == ret) {
snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "\"%s\",", buf);
} else {
snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s,", buf);
}
if (i == num_fields - 1) {
sprintf(result + strlen(result) - 1, "%s", ")");
}
}
if (0 == strlen(result)) {
return TSDB_CODE_TSC_INVALID_TABLE_NAME;
}
return TSDB_CODE_SUCCESS;
}
// build 'show create table/database' result fields
static int32_t tscSCreateBuildResultFields(SSqlObj *pSql, BuildType type, const char *ddl) {
int32_t rowLen = 0;
int16_t ddlLen = (int16_t)strlen(ddl);
SColumnIndex index = {0};
pSql->cmd.numOfCols = 2;
SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd);
pQueryInfo->order.order = TSDB_ORDER_ASC;
TAOS_FIELD f;
if (type == SCREATE_BUILD_TABLE) {
f.type = TSDB_DATA_TYPE_BINARY;
f.bytes = (TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE;
tstrncpy(f.name, "Table", sizeof(f.name));
} else {
f.type = TSDB_DATA_TYPE_BINARY;
f.bytes = (TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE;
tstrncpy(f.name, "Database", sizeof(f.name));
}
SInternalField* pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, f.bytes, -1000, f.bytes - VARSTR_HEADER_SIZE, false);
rowLen += f.bytes;
f.bytes = (int16_t)(ddlLen + VARSTR_HEADER_SIZE);
f.type = TSDB_DATA_TYPE_BINARY;
if (type == SCREATE_BUILD_TABLE) {
tstrncpy(f.name, "Create Table", sizeof(f.name));
} else {
tstrncpy(f.name, "Create Database", sizeof(f.name));
}
pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY,
(int16_t)(ddlLen + VARSTR_HEADER_SIZE), -1000, ddlLen, false);
rowLen += ddlLen + VARSTR_HEADER_SIZE;
return rowLen;
}
static int32_t tscSCreateSetValueToResObj(SSqlObj *pSql, int32_t rowLen, const char *tableName, const char *ddl) {
SSqlRes *pRes = &pSql->res;
SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd);
int32_t numOfRows = 1;
if (strlen(ddl) == 0) {
}
pSql->res.pMerger = tscInitResObjForLocalQuery(numOfRows, rowLen, pSql->self);
tscInitResForMerge(&pSql->res);
TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0);
char* dst = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 0) * numOfRows;
STR_WITH_MAXSIZE_TO_VARSTR(dst, tableName, pField->bytes);
pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 1);
dst = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 1) * numOfRows;
STR_WITH_MAXSIZE_TO_VARSTR(dst, ddl, pField->bytes);
return 0;
}
static int32_t tscSCreateBuildResult(SSqlObj *pSql, BuildType type, const char *str, const char *result) {
SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd);
int32_t rowLen = tscSCreateBuildResultFields(pSql, type, result);
tscFieldInfoUpdateOffset(pQueryInfo);
return tscSCreateSetValueToResObj(pSql, rowLen, str, result);
}
int32_t tscRebuildCreateTableStatement(void *param,char *result) {
SCreateBuilder *builder = (SCreateBuilder *)param;
int32_t code = TSDB_CODE_SUCCESS;
char *buf = calloc(1,TSDB_MAX_BINARY_LEN);
if (buf == NULL) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
code = tscGetTableTagValue(builder, buf);
if (code == TSDB_CODE_SUCCESS) {
snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "CREATE TABLE %s USING %s TAGS %s", builder->buf, builder->sTableName, buf);
code = tscSCreateBuildResult(builder->pParentSql, SCREATE_BUILD_TABLE, builder->buf, result);
}
free(buf);
return code;
}
static int32_t tscGetDBInfo(SCreateBuilder *builder, char *result) {
TAOS_ROW row = tscFetchRow(builder);
if (row == NULL) {
return TSDB_CODE_TSC_DB_NOT_SELECTED;
}
const char *showColumns[] = {"REPLICA", "QUORUM", "DAYS", "KEEP", "BLOCKS", NULL};
SSqlObj *pSql = builder->pInterSql;
TAOS_FIELD *fields = taos_fetch_fields(pSql);
int num_fields = taos_num_fields(pSql);
char buf[TSDB_DB_NAME_LEN + 64] = {0};
do {
memset(buf, 0, sizeof(buf));
int32_t* lengths = taos_fetch_lengths(pSql);
int32_t ret = tscGetNthFieldResult(row, fields, lengths, 0, buf);
if (0 == ret && STR_NOCASE_EQUAL(buf, strlen(buf), builder->buf, strlen(builder->buf))) {
snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "CREATE DATABASE %s", buf);
for (int i = 1; i < num_fields; i++) {
for (int j = 0; showColumns[j] != NULL; j++) {
if (STR_NOCASE_EQUAL(fields[i].name, strlen(fields[i].name), showColumns[j], strlen(showColumns[j]))) {
memset(buf, 0, sizeof(buf));
ret = tscGetNthFieldResult(row, fields, lengths, i, buf);
if (ret == 0) {
snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), " %s %s", showColumns[j], buf);
}
}
}
}
break;
}
row = tscFetchRow(builder);
} while (row != NULL);
if (0 == strlen(result)) {
return TSDB_CODE_TSC_DB_NOT_SELECTED;
}
return TSDB_CODE_SUCCESS;
}
int32_t tscRebuildCreateDBStatement(void *param,char *result) {
SCreateBuilder *builder = (SCreateBuilder *)param;
int32_t code = TSDB_CODE_SUCCESS;
char *buf = calloc(1, TSDB_MAX_BINARY_LEN);
if (buf == NULL) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
code = tscGetDBInfo(param, buf);
if (code == TSDB_CODE_SUCCESS) {
code = tscSCreateBuildResult(builder->pParentSql, SCREATE_BUILD_DB, builder->buf, buf);
}
free(buf);
return code;
}
static int32_t tscGetTableTagColumnName(SSqlObj *pSql, char **result) {
char *buf = (char *)malloc(TSDB_MAX_BINARY_LEN);
if (buf == NULL) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
buf[0] = 0;
STableMeta *pMeta = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0)->pTableMeta;
if (pMeta->tableType == TSDB_SUPER_TABLE || pMeta->tableType == TSDB_NORMAL_TABLE ||
pMeta->tableType == TSDB_STREAM_TABLE) {
free(buf);
return TSDB_CODE_TSC_INVALID_VALUE;
}
SSchema *pTagsSchema = tscGetTableTagSchema(pMeta);
int32_t numOfTags = tscGetNumOfTags(pMeta);
for (int32_t i = 0; i < numOfTags; i++) {
if (i != numOfTags - 1) {
snprintf(buf + strlen(buf), TSDB_MAX_BINARY_LEN - strlen(buf), "%s,", pTagsSchema[i].name);
} else {
snprintf(buf + strlen(buf), TSDB_MAX_BINARY_LEN - strlen(buf), "%s", pTagsSchema[i].name);
}
}
*result = buf;
return TSDB_CODE_SUCCESS;
}
static int32_t tscRebuildDDLForSubTable(SSqlObj *pSql, const char *tableName, char *ddl) {
SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd);
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
STableMeta * pMeta = pTableMetaInfo->pTableMeta;
SSqlObj *pInterSql = (SSqlObj *)calloc(1, sizeof(SSqlObj));
if (pInterSql == NULL) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
SCreateBuilder *param = (SCreateBuilder *)malloc(sizeof(SCreateBuilder));
if (param == NULL) {
free(pInterSql);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
char fullName[TSDB_TABLE_FNAME_LEN * 2] = {0};
tNameGetDbName(&pTableMetaInfo->name, fullName);
extractTableName(pMeta->sTableName, param->sTableName);
snprintf(fullName + strlen(fullName), TSDB_TABLE_FNAME_LEN - strlen(fullName), ".%s", param->sTableName);
strncpy(param->buf, tNameGetTableName(&pTableMetaInfo->name), TSDB_TABLE_NAME_LEN);
param->pParentSql = pSql;
param->pInterSql = pInterSql;
param->fp = tscRebuildCreateTableStatement;
param->callStage = SCREATE_CALLBACK_QUERY;
char *query = (char *)calloc(1, TSDB_MAX_BINARY_LEN);
if (query == NULL) {
free(param);
free(pInterSql);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
char *columns = NULL;
int32_t code = tscGetTableTagColumnName(pSql, &columns) ;
if (code != TSDB_CODE_SUCCESS) {
free(param);
free(pInterSql);
free(query);
return code;
}
snprintf(query + strlen(query), TSDB_MAX_BINARY_LEN - strlen(query), "SELECT %s FROM %s WHERE TBNAME IN(\'%s\')", columns, fullName, param->buf);
doAsyncQuery(pSql->pTscObj, pInterSql, tscSCreateCallBack, param, query, strlen(query));
free(query);
free(columns);
return TSDB_CODE_TSC_ACTION_IN_PROGRESS;
}
static int32_t tscRebuildDDLForNormalTable(SSqlObj *pSql, const char *tableName, char *ddl) {
SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd);
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
STableMeta * pMeta = pTableMetaInfo->pTableMeta;
int32_t numOfRows = tscGetNumOfColumns(pMeta);
SSchema *pSchema = tscGetTableSchema(pMeta);
char *result = ddl;
sprintf(result, "create table %s (", tableName);
for (int32_t i = 0; i < numOfRows; ++i) {
uint8_t type = pSchema[i].type;
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
int32_t bytes = pSchema[i].bytes - VARSTR_HEADER_SIZE;
if (type == TSDB_DATA_TYPE_NCHAR) {
bytes = bytes/TSDB_NCHAR_SIZE;
}
snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s(%d),", pSchema[i].name, tDataTypes[pSchema[i].type].name, bytes);
} else {
snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s,", pSchema[i].name, tDataTypes[pSchema[i].type].name);
}
}
sprintf(result + strlen(result) - 1, "%s", ")");
return TSDB_CODE_SUCCESS;
}
static int32_t tscRebuildDDLForSuperTable(SSqlObj *pSql, const char *tableName, char *ddl) {
char *result = ddl;
SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd);
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
STableMeta * pMeta = pTableMetaInfo->pTableMeta;
int32_t numOfRows = tscGetNumOfColumns(pMeta);
int32_t totalRows = numOfRows + tscGetNumOfTags(pMeta);
SSchema *pSchema = tscGetTableSchema(pMeta);
sprintf(result, "create table %s (", tableName);
for (int32_t i = 0; i < numOfRows; ++i) {
uint8_t type = pSchema[i].type;
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
int32_t bytes = pSchema[i].bytes - VARSTR_HEADER_SIZE;
if (type == TSDB_DATA_TYPE_NCHAR) {
bytes = bytes/TSDB_NCHAR_SIZE;
}
snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result),"%s %s(%d),", pSchema[i].name,tDataTypes[pSchema[i].type].name, bytes);
} else {
snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s,", pSchema[i].name, tDataTypes[type].name);
}
}
snprintf(result + strlen(result) - 1, TSDB_MAX_BINARY_LEN - strlen(result), "%s %s", ")", "TAGS (");
for (int32_t i = numOfRows; i < totalRows; i++) {
uint8_t type = pSchema[i].type;
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
int32_t bytes = pSchema[i].bytes - VARSTR_HEADER_SIZE;
if (type == TSDB_DATA_TYPE_NCHAR) {
bytes = bytes/TSDB_NCHAR_SIZE;
}
snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s(%d),", pSchema[i].name,tDataTypes[pSchema[i].type].name, bytes);
} else {
snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s,", pSchema[i].name, tDataTypes[type].name);
}
}
sprintf(result + strlen(result) - 1, "%s", ")");
return TSDB_CODE_SUCCESS;
}
static int32_t tscProcessShowCreateTable(SSqlObj *pSql) {
SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd);
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
assert(pTableMetaInfo->pTableMeta != NULL);
const char* tableName = tNameGetTableName(&pTableMetaInfo->name);
if (pSql->cmd.command == TSDB_SQL_SHOW_CREATE_STABLE && !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
return TSDB_CODE_TSC_INVALID_VALUE;
}
char *result = (char *)calloc(1, TSDB_MAX_BINARY_LEN);
int32_t code = TSDB_CODE_SUCCESS;
if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
code = tscRebuildDDLForSuperTable(pSql, tableName, result);
} else if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) {
code = tscRebuildDDLForNormalTable(pSql, tableName, result);
} else if (UTIL_TABLE_IS_CHILD_TABLE(pTableMetaInfo)) {
code = tscRebuildDDLForSubTable(pSql, tableName, result);
} else {
code = TSDB_CODE_TSC_INVALID_VALUE;
}
if (code == TSDB_CODE_SUCCESS) {
code = tscSCreateBuildResult(pSql, SCREATE_BUILD_TABLE, tableName, result);
}
free(result);
return code;
}
static int32_t tscProcessShowCreateDatabase(SSqlObj *pSql) {
SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd);
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
SSqlObj *pInterSql = (SSqlObj *)calloc(1, sizeof(SSqlObj));
if (pInterSql == NULL) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
SCreateBuilder *param = (SCreateBuilder *)calloc(1, sizeof(SCreateBuilder));
if (param == NULL) {
free(pInterSql);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
tNameGetDbName(&pTableMetaInfo->name, param->buf);
param->pParentSql = pSql;
param->pInterSql = pInterSql;
param->fp = tscRebuildCreateDBStatement;
param->callStage = SCREATE_CALLBACK_QUERY;
const char *query = "show databases";
doAsyncQuery(pSql->pTscObj, pInterSql, tscSCreateCallBack, param, query, strlen(query));
return TSDB_CODE_TSC_ACTION_IN_PROGRESS;
}
static int32_t tscProcessCurrentUser(SSqlObj *pSql) {
SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd);
SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0);
pExpr->resBytes = TSDB_USER_LEN + TSDB_DATA_TYPE_BINARY;
pExpr->resType = TSDB_DATA_TYPE_BINARY;
char* vx = calloc(1, pExpr->resBytes);
if (vx == NULL) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
size_t size = sizeof(pSql->pTscObj->user);
STR_WITH_MAXSIZE_TO_VARSTR(vx, pSql->pTscObj->user, size);
tscSetLocalQueryResult(pSql, vx, pExpr->aliasName, pExpr->resType, pExpr->resBytes);
free(vx);
return TSDB_CODE_SUCCESS;
}
static int32_t tscProcessCurrentDB(SSqlObj *pSql) {
char db[TSDB_DB_NAME_LEN] = {0};
pthread_mutex_lock(&pSql->pTscObj->mutex);
extractDBName(pSql->pTscObj->db, db);
pthread_mutex_unlock(&pSql->pTscObj->mutex);
SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd);
SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0);
pExpr->resType = TSDB_DATA_TYPE_BINARY;
size_t t = strlen(db);
pExpr->resBytes = TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE;
char* vx = calloc(1, pExpr->resBytes);
if (vx == NULL) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
if (t == 0) {
setVardataNull(vx, TSDB_DATA_TYPE_BINARY);
} else {
STR_WITH_SIZE_TO_VARSTR(vx, db, (VarDataLenT)t);
}
tscSetLocalQueryResult(pSql, vx, pExpr->aliasName, pExpr->resType, pExpr->resBytes);
free(vx);
return TSDB_CODE_SUCCESS;
}
static int32_t tscProcessServerVer(SSqlObj *pSql) {
const char* v = pSql->pTscObj->sversion;
SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd);
SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0);
pExpr->resType = TSDB_DATA_TYPE_BINARY;
size_t t = strlen(v);
pExpr->resBytes = (int16_t)(t + VARSTR_HEADER_SIZE);
char* vx = calloc(1, pExpr->resBytes);
if (vx == NULL) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
STR_WITH_SIZE_TO_VARSTR(vx, v, (VarDataLenT)t);
tscSetLocalQueryResult(pSql, vx, pExpr->aliasName, pExpr->resType, pExpr->resBytes);
free(vx);
return TSDB_CODE_SUCCESS;
}
static int32_t tscProcessClientVer(SSqlObj *pSql) {
SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd);
SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0);
pExpr->resType = TSDB_DATA_TYPE_BINARY;
size_t t = strlen(version);
pExpr->resBytes = (int16_t)(t + VARSTR_HEADER_SIZE);
char* v = calloc(1, pExpr->resBytes);
if (v == NULL) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
STR_WITH_SIZE_TO_VARSTR(v, version, (VarDataLenT)t);
tscSetLocalQueryResult(pSql, v, pExpr->aliasName, pExpr->resType, pExpr->resBytes);
free(v);
return TSDB_CODE_SUCCESS;
}
// TODO add test cases.
static int32_t checkForOnlineNode(SSqlObj* pSql) {
int32_t* data = pSql->res.length;
if (data == NULL) {
return TSDB_CODE_SUCCESS;
}
int32_t total = data[0];
int32_t online = data[1];
return (online < total)? TSDB_CODE_RPC_NETWORK_UNAVAIL:TSDB_CODE_SUCCESS;
}
static int32_t tscProcessServStatus(SSqlObj *pSql) {
STscObj* pObj = pSql->pTscObj;
SSqlObj* pHb = (SSqlObj*)taosAcquireRef(tscObjRef, pObj->hbrid);
if (pHb != NULL) {
pSql->res.code = pHb->res.code;
}
if (pSql->res.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
taosReleaseRef(tscObjRef, pObj->hbrid);
return pSql->res.code;
}
if (pHb != NULL) {
pSql->res.code = checkForOnlineNode(pHb);
taosReleaseRef(tscObjRef, pObj->hbrid);
}
if (pSql->res.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
return pSql->res.code;
}
SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd);
SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0);
int32_t val = 1;
tscSetLocalQueryResult(pSql, (char*) &val, pExpr->aliasName, TSDB_DATA_TYPE_INT, sizeof(int32_t));
return TSDB_CODE_SUCCESS;
}
void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnName, int16_t type, size_t valueLength) {
SSqlCmd *pCmd = &pSql->cmd;
SSqlRes *pRes = &pSql->res;
pCmd->numOfCols = 1;
SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd);
pQueryInfo->order.order = TSDB_ORDER_ASC;
tscFieldInfoClear(&pQueryInfo->fieldsInfo);
pQueryInfo->fieldsInfo.internalField = taosArrayInit(1, sizeof(SInternalField));
TAOS_FIELD f = tscCreateField((int8_t)type, columnName, (int16_t)valueLength);
tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
pSql->res.pMerger = tscInitResObjForLocalQuery(1, (int32_t)valueLength, pSql->self);
tscInitResForMerge(&pSql->res);
SInternalField* pInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, 0);
pInfo->pExpr = taosArrayGetP(pQueryInfo->exprList, 0);
memcpy(pRes->data, val, pInfo->field.bytes);
}
int tscProcessLocalCmd(SSqlObj *pSql) {
SSqlCmd *pCmd = &pSql->cmd;
SSqlRes *pRes = &pSql->res;
if (pCmd->command == TSDB_SQL_CFG_LOCAL) {
if (taosCfgDynamicOptions(pCmd->payload)) {
pRes->code = TSDB_CODE_SUCCESS;
} else {
pRes->code = TSDB_CODE_COM_INVALID_CFG_MSG;
}
pRes->numOfRows = 0;
} else if (pCmd->command == TSDB_SQL_DESCRIBE_TABLE) {
pRes->code = (uint8_t)tscProcessDescribeTable(pSql);
} else if (pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT) {
/*
* set the qhandle to be 1 in order to pass the qhandle check, and to call partial release function to
* free allocated resources and remove the SqlObj from sql query linked list
*/
pRes->qId = 0x1;
pRes->numOfRows = 0;
} else if (pCmd->command == TSDB_SQL_SHOW_CREATE_TABLE || pCmd->command == TSDB_SQL_SHOW_CREATE_STABLE) {
pRes->code = tscProcessShowCreateTable(pSql);
} else if (pCmd->command == TSDB_SQL_SHOW_CREATE_DATABASE) {
pRes->code = tscProcessShowCreateDatabase(pSql);
} else if (pCmd->command == TSDB_SQL_RESET_CACHE) {
taosHashClear(tscTableMetaMap);
taosCacheEmpty(tscVgroupListBuf);
pRes->code = TSDB_CODE_SUCCESS;
} else if (pCmd->command == TSDB_SQL_SERV_VERSION) {
pRes->code = tscProcessServerVer(pSql);
} else if (pCmd->command == TSDB_SQL_CLI_VERSION) {
pRes->code = tscProcessClientVer(pSql);
} else if (pCmd->command == TSDB_SQL_CURRENT_USER) {
pRes->code = tscProcessCurrentUser(pSql);
} else if (pCmd->command == TSDB_SQL_CURRENT_DB) {
pRes->code = tscProcessCurrentDB(pSql);
} else if (pCmd->command == TSDB_SQL_SERV_STATUS) {
pRes->code = tscProcessServStatus(pSql);
} else {
pRes->code = TSDB_CODE_TSC_INVALID_OPERATION;
tscError("0x%"PRIx64" not support command:%d", pSql->self, pCmd->command);
}
// keep the code in local variable in order to avoid invalid read in case of async query
int32_t code = pRes->code;
if (code == TSDB_CODE_SUCCESS) {
(*pSql->fp)(pSql->param, pSql, code);
} else if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS){
} else {
tscAsyncResultOnError(pSql);
}
return code;
}

1803
2.0/src/client/src/tscParseInsert.c
View File

File diff suppressed because it is too large Load Diff

2184
2.0/src/client/src/tscParseLineProtocol.c
View File

File diff suppressed because it is too large Load Diff

973
2.0/src/client/src/tscParseOpenTSDB.c
View File

@ -1,973 +0,0 @@
#include <ctype.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "../../../include/client/taos.h"
#include "cJSON.h"
#include "hash.h"
#include "tscUtil.h"
#include "tsclient.h"
#include "tscLog.h"
#include "tscParseLine.h"
#define OTD_MAX_FIELDS_NUM 2
#define OTD_JSON_SUB_FIELDS_NUM 2
#define OTD_JSON_FIELDS_NUM 4
#define OTD_TIMESTAMP_COLUMN_NAME "ts"
#define OTD_METRIC_VALUE_COLUMN_NAME "value"
/* telnet style API parser */
static uint64_t HandleId = 0;
static uint64_t genUID() {
uint64_t id;
do {
id = atomic_add_fetch_64(&HandleId, 1);
} while (id == 0);
return id;
}
static int32_t parseTelnetMetric(TAOS_SML_DATA_POINT *pSml, const char **index, SSmlLinesInfo* info) {
const char *cur = *index;
uint16_t len = 0;
pSml->stableName = tcalloc(TSDB_TABLE_NAME_LEN + 1, 1); // +1 to avoid 1772 line over write
if (pSml->stableName == NULL){
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
if (isdigit(*cur)) {
tscError("OTD:0x%"PRIx64" Metric cannnot start with digit", info->id);
tfree(pSml->stableName);
return TSDB_CODE_TSC_LINE_SYNTAX_ERROR;
}
while (*cur != '\0') {
if (len > TSDB_TABLE_NAME_LEN) {
tscError("OTD:0x%"PRIx64" Metric cannot exceeds 193 characters", info->id);
tfree(pSml->stableName);
return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH;
}
if (*cur == ' ') {
break;
}
pSml->stableName[len] = *cur;
cur++;
len++;
}
if (len == 0 || *cur == '\0') {
tfree(pSml->stableName);
return TSDB_CODE_TSC_LINE_SYNTAX_ERROR;
}
pSml->stableName[len] = '\0';
*index = cur + 1;
tscDebug("OTD:0x%"PRIx64" Stable name in metric:%s|len:%d", info->id, pSml->stableName, len);
return TSDB_CODE_SUCCESS;
}
static int32_t parseTelnetTimeStamp(TAOS_SML_KV **pTS, int *num_kvs, const char **index, SSmlLinesInfo* info) {
//Timestamp must be the first KV to parse
assert(*num_kvs == 0);
const char *start, *cur;
int32_t ret = TSDB_CODE_SUCCESS;
int len = 0;
char key[] = OTD_TIMESTAMP_COLUMN_NAME;
char *value = NULL;
start = cur = *index;
//allocate fields for timestamp and value
*pTS = tcalloc(OTD_MAX_FIELDS_NUM, sizeof(TAOS_SML_KV));
while(*cur != '\0') {
if (*cur == ' ') {
break;
}
cur++;
len++;
}
if (len > 0 && *cur != '\0') {
value = tcalloc(len + 1, 1);
memcpy(value, start, len);
} else {
tfree(*pTS);
return TSDB_CODE_TSC_LINE_SYNTAX_ERROR;
}
ret = convertSmlTimeStamp(*pTS, value, len, info);
if (ret) {
tfree(value);
tfree(*pTS);
return ret;
}
tfree(value);
(*pTS)->key = tcalloc(sizeof(key), 1);
memcpy((*pTS)->key, key, sizeof(key));
*num_kvs += 1;
*index = cur + 1;
return ret;
}
static int32_t parseTelnetMetricValue(TAOS_SML_KV **pKVs, int *num_kvs, const char **index, SSmlLinesInfo* info) {
//skip timestamp
TAOS_SML_KV *pVal = *pKVs + 1;
const char *start, *cur;
int32_t ret = TSDB_CODE_SUCCESS;
int len = 0;
char key[] = OTD_METRIC_VALUE_COLUMN_NAME;
char *value = NULL;
start = cur = *index;
while(*cur != '\0') {
if (*cur == ' ') {
break;
}
cur++;
len++;
}
if (len > 0 && *cur != '\0') {
value = tcalloc(len + 1, 1);
memcpy(value, start, len);
} else {
return TSDB_CODE_TSC_LINE_SYNTAX_ERROR;
}
if (!convertSmlValueType(pVal, value, len, info)) {
tscError("OTD:0x%"PRIx64" Failed to convert metric value string(%s) to any type",
info->id, value);
tfree(value);
return TSDB_CODE_TSC_INVALID_VALUE;
}
tfree(value);
pVal->key = tcalloc(sizeof(key), 1);
memcpy(pVal->key, key, sizeof(key));
*num_kvs += 1;
*index = cur + 1;
return ret;
}
static int32_t parseTelnetTagKey(TAOS_SML_KV *pKV, const char **index, SHashObj *pHash, SSmlLinesInfo* info) {
const char *cur = *index;
char key[TSDB_COL_NAME_LEN + 1]; // +1 to avoid key[len] over write
uint16_t len = 0;
//key field cannot start with digit
if (isdigit(*cur)) {
tscError("OTD:0x%"PRIx64" Tag key cannnot start with digit", info->id);
return TSDB_CODE_TSC_LINE_SYNTAX_ERROR;
}
while (*cur != '\0') {
if (len > TSDB_COL_NAME_LEN) {
tscError("OTD:0x%"PRIx64" Tag key cannot exceeds 65 characters", info->id);
return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
}
if (*cur == '=') {
break;
}
key[len] = *cur;
cur++;
len++;
}
if (len == 0 || *cur == '\0') {
return TSDB_CODE_TSC_LINE_SYNTAX_ERROR;
}
key[len] = '\0';
if (checkDuplicateKey(key, pHash, info)) {
return TSDB_CODE_TSC_DUP_TAG_NAMES;
}
pKV->key = tcalloc(len + 1, 1);
memcpy(pKV->key, key, len + 1);
//tscDebug("OTD:0x%"PRIx64" Key:%s|len:%d", info->id, pKV->key, len);
*index = cur + 1;
return TSDB_CODE_SUCCESS;
}
static int32_t parseTelnetTagValue(TAOS_SML_KV *pKV, const char **index,
bool *is_last_kv, SSmlLinesInfo* info) {
const char *start, *cur;
char *value = NULL;
uint16_t len = 0;
start = cur = *index;
while (1) {
// ',' or '\0' identifies a value
if (*cur == ',' || *cur == '\0') {
// '\0' indicates end of value
*is_last_kv = (*cur == '\0') ? true : false;
break;
}
cur++;
len++;
}
if (len == 0) {
tfree(pKV->key);
return TSDB_CODE_TSC_LINE_SYNTAX_ERROR;
}
value = tcalloc(len + 1, 1);
memcpy(value, start, len);
value[len] = '\0';
if (!convertSmlValueType(pKV, value, len, info)) {
tscError("OTD:0x%"PRIx64" Failed to convert sml value string(%s) to any type",
info->id, value);
//free previous alocated key field
tfree(pKV->key);
tfree(value);
return TSDB_CODE_TSC_INVALID_VALUE;
}
tfree(value);
*index = (*cur == '\0') ? cur : cur + 1;
return TSDB_CODE_SUCCESS;
}
static int32_t parseTelnetTagKvs(TAOS_SML_KV **pKVs, int *num_kvs,
const char **index, char **childTableName,
SHashObj *pHash, SSmlLinesInfo* info) {
const char *cur = *index;
int32_t ret = TSDB_CODE_SUCCESS;
TAOS_SML_KV *pkv;
bool is_last_kv = false;
int32_t capacity = 4;
*pKVs = tcalloc(capacity, sizeof(TAOS_SML_KV));
pkv = *pKVs;
while (*cur != '\0') {
ret = parseTelnetTagKey(pkv, &cur, pHash, info);
if (ret) {
tscError("OTD:0x%"PRIx64" Unable to parse key", info->id);
return ret;
}
ret = parseTelnetTagValue(pkv, &cur, &is_last_kv, info);
if (ret) {
tscError("OTD:0x%"PRIx64" Unable to parse value", info->id);
return ret;
}
if ((strcasecmp(pkv->key, "ID") == 0) && pkv->type == TSDB_DATA_TYPE_BINARY) {
ret = isValidChildTableName(pkv->value, pkv->length);
if (ret) {
return ret;
}
*childTableName = malloc(pkv->length + 1);
memcpy(*childTableName, pkv->value, pkv->length);
(*childTableName)[pkv->length] = '\0';
tfree(pkv->key);
tfree(pkv->value);
} else {
*num_kvs += 1;
}
if (is_last_kv) {
break;
}
//reallocate addtional memory for more kvs
if ((*num_kvs + 1) > capacity) {
TAOS_SML_KV *more_kvs = NULL;
capacity *= 3; capacity /= 2;
more_kvs = realloc(*pKVs, capacity * sizeof(TAOS_SML_KV));
if (!more_kvs) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
*pKVs = more_kvs;
}
//move pKV points to next TAOS_SML_KV block
pkv = *pKVs + *num_kvs;
}
return ret;
}
int32_t tscParseTelnetLine(const char* line, TAOS_SML_DATA_POINT* smlData, SSmlLinesInfo* info) {
const char* index = line;
int32_t ret = TSDB_CODE_SUCCESS;
//Parse metric
ret = parseTelnetMetric(smlData, &index, info);
if (ret) {
tscError("OTD:0x%"PRIx64" Unable to parse metric", info->id);
return ret;
}
tscDebug("OTD:0x%"PRIx64" Parse metric finished", info->id);
//Parse timestamp
ret = parseTelnetTimeStamp(&smlData->fields, &smlData->fieldNum, &index, info);
if (ret) {
tscError("OTD:0x%"PRIx64" Unable to parse timestamp", info->id);
return ret;
}
tscDebug("OTD:0x%"PRIx64" Parse timestamp finished", info->id);
//Parse value
ret = parseTelnetMetricValue(&smlData->fields, &smlData->fieldNum, &index, info);
if (ret) {
tscError("OTD:0x%"PRIx64" Unable to parse metric value", info->id);
return ret;
}
tscDebug("OTD:0x%"PRIx64" Parse metric value finished", info->id);
//Parse tagKVs
SHashObj *keyHashTable = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false);
ret = parseTelnetTagKvs(&smlData->tags, &smlData->tagNum, &index, &smlData->childTableName, keyHashTable, info);
if (ret) {
tscError("OTD:0x%"PRIx64" Unable to parse tags", info->id);
taosHashCleanup(keyHashTable);
return ret;
}
tscDebug("OTD:0x%"PRIx64" Parse tags finished", info->id);
taosHashCleanup(keyHashTable);
return TSDB_CODE_SUCCESS;
}
int32_t tscParseTelnetLines(char* lines[], int numLines, SArray* points, SArray* failedLines, SSmlLinesInfo* info) {
for (int32_t i = 0; i < numLines; ++i) {
TAOS_SML_DATA_POINT point = {0};
int32_t code = tscParseTelnetLine(lines[i], &point, info);
if (code != TSDB_CODE_SUCCESS) {
tscError("OTD:0x%"PRIx64" data point line parse failed. line %d : %s", info->id, i, lines[i]);
destroySmlDataPoint(&point);
return code;
} else {
tscDebug("OTD:0x%"PRIx64" data point line parse success. line %d", info->id, i);
}
taosArrayPush(points, &point);
}
return TSDB_CODE_SUCCESS;
}
int taos_insert_telnet_lines(TAOS* taos, char* lines[], int numLines) {
int32_t code = 0;
SSmlLinesInfo* info = tcalloc(1, sizeof(SSmlLinesInfo));
info->id = genUID();
if (numLines <= 0 || numLines > 65536) {
tscError("OTD:0x%"PRIx64" taos_insert_telnet_lines numLines should be between 1 and 65536. numLines: %d", info->id, numLines);
tfree(info);
code = TSDB_CODE_TSC_APP_ERROR;
return code;
}
for (int i = 0; i < numLines; ++i) {
if (lines[i] == NULL) {
tscError("OTD:0x%"PRIx64" taos_insert_telnet_lines line %d is NULL", info->id, i);
tfree(info);
code = TSDB_CODE_TSC_APP_ERROR;
return code;
}
}
SArray* lpPoints = taosArrayInit(numLines, sizeof(TAOS_SML_DATA_POINT));
if (lpPoints == NULL) {
tscError("OTD:0x%"PRIx64" taos_insert_telnet_lines failed to allocate memory", info->id);
tfree(info);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
tscDebug("OTD:0x%"PRIx64" taos_insert_telnet_lines begin inserting %d lines, first line: %s", info->id, numLines, lines[0]);
code = tscParseTelnetLines(lines, numLines, lpPoints, NULL, info);
size_t numPoints = taosArrayGetSize(lpPoints);
if (code != 0) {
goto cleanup;
}
TAOS_SML_DATA_POINT* points = TARRAY_GET_START(lpPoints);
code = tscSmlInsert(taos, points, (int)numPoints, info);
if (code != 0) {
tscError("OTD:0x%"PRIx64" taos_insert_telnet_lines error: %s", info->id, tstrerror((code)));
}
cleanup:
tscDebug("OTD:0x%"PRIx64" taos_insert_telnet_lines finish inserting %d lines. code: %d", info->id, numLines, code);
points = TARRAY_GET_START(lpPoints);
numPoints = taosArrayGetSize(lpPoints);
for (int i = 0; i < numPoints; ++i) {
destroySmlDataPoint(points+i);
}
taosArrayDestroy(lpPoints);
tfree(info);
return code;
}
int taos_telnet_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) {
SSmlLinesInfo* info = tcalloc(1, sizeof(SSmlLinesInfo));
info->id = genUID();
int code = tscSmlInsert(taos, points, numPoint, info);
tfree(info);
return code;
}
/* telnet style API parser */
int32_t parseMetricFromJSON(cJSON *root, TAOS_SML_DATA_POINT* pSml, SSmlLinesInfo* info) {
cJSON *metric = cJSON_GetObjectItem(root, "metric");
if (!cJSON_IsString(metric)) {
return TSDB_CODE_TSC_INVALID_JSON;
}
size_t stableLen = strlen(metric->valuestring);
if (stableLen > TSDB_TABLE_NAME_LEN) {
tscError("OTD:0x%"PRIx64" Metric cannot exceeds 193 characters in JSON", info->id);
return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH;
}
pSml->stableName = tcalloc(stableLen + 1, sizeof(char));
if (pSml->stableName == NULL){
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
if (isdigit(metric->valuestring[0])) {
tscError("OTD:0x%"PRIx64" Metric cannnot start with digit in JSON", info->id);
tfree(pSml->stableName);
return TSDB_CODE_TSC_INVALID_JSON;
}
tstrncpy(pSml->stableName, metric->valuestring, stableLen + 1);
return TSDB_CODE_SUCCESS;
}
int32_t parseTimestampFromJSONObj(cJSON *root, int64_t *tsVal, SSmlLinesInfo* info) {
int32_t size = cJSON_GetArraySize(root);
if (size != OTD_JSON_SUB_FIELDS_NUM) {
return TSDB_CODE_TSC_INVALID_JSON;
}
cJSON *value = cJSON_GetObjectItem(root, "value");
if (!cJSON_IsNumber(value)) {
return TSDB_CODE_TSC_INVALID_JSON;
}
cJSON *type = cJSON_GetObjectItem(root, "type");
if (!cJSON_IsString(type)) {
return TSDB_CODE_TSC_INVALID_JSON;
}
*tsVal = value->valueint;
//if timestamp value is 0 use current system time
if (*tsVal == 0) {
*tsVal = taosGetTimestampNs();
return TSDB_CODE_SUCCESS;
}
size_t typeLen = strlen(type->valuestring);
if (typeLen == 1 && type->valuestring[0] == 's') {
//seconds
*tsVal = (int64_t)(*tsVal * 1e9);
} else if (typeLen == 2 && type->valuestring[1] == 's') {
switch (type->valuestring[0]) {
case 'm':
//milliseconds
*tsVal = convertTimePrecision(*tsVal, TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_NANO);
break;
case 'u':
//microseconds
*tsVal = convertTimePrecision(*tsVal, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO);
break;
case 'n':
//nanoseconds
*tsVal = *tsVal * 1;
break;
default:
return TSDB_CODE_TSC_INVALID_JSON;
}
}
return TSDB_CODE_SUCCESS;
}
int32_t parseTimestampFromJSON(cJSON *root, TAOS_SML_KV **pTS, int *num_kvs, SSmlLinesInfo* info) {
//Timestamp must be the first KV to parse
assert(*num_kvs == 0);
int64_t tsVal;
char key[] = OTD_TIMESTAMP_COLUMN_NAME;
cJSON *timestamp = cJSON_GetObjectItem(root, "timestamp");
if (cJSON_IsNumber(timestamp)) {
//timestamp value 0 indicates current system time
if (timestamp->valueint == 0) {
tsVal = taosGetTimestampNs();
} else {
tsVal = convertTimePrecision(timestamp->valueint, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO);
}
} else if (cJSON_IsObject(timestamp)) {
int32_t ret = parseTimestampFromJSONObj(timestamp, &tsVal, info);
if (ret != TSDB_CODE_SUCCESS) {
tscError("OTD:0x%"PRIx64" Failed to parse timestamp from JSON Obj", info->id);
return ret;
}
} else {
return TSDB_CODE_TSC_INVALID_JSON;
}
//allocate fields for timestamp and value
*pTS = tcalloc(OTD_MAX_FIELDS_NUM, sizeof(TAOS_SML_KV));
(*pTS)->key = tcalloc(sizeof(key), 1);
memcpy((*pTS)->key, key, sizeof(key));
(*pTS)->type = TSDB_DATA_TYPE_TIMESTAMP;
(*pTS)->length = (int16_t)tDataTypes[(*pTS)->type].bytes;
(*pTS)->value = tcalloc((*pTS)->length, 1);
memcpy((*pTS)->value, &tsVal, (*pTS)->length);
*num_kvs += 1;
return TSDB_CODE_SUCCESS;
}
int32_t convertJSONBool(TAOS_SML_KV *pVal, char* typeStr, int64_t valueInt, SSmlLinesInfo* info) {
if (strcasecmp(typeStr, "bool") != 0) {
tscError("OTD:0x%"PRIx64" invalid type(%s) for JSON Bool", info->id, typeStr);
return TSDB_CODE_TSC_INVALID_JSON_TYPE;
}
pVal->type = TSDB_DATA_TYPE_BOOL;
pVal->length = (int16_t)tDataTypes[pVal->type].bytes;
pVal->value = tcalloc(pVal->length, 1);
*(bool *)(pVal->value) = valueInt ? true : false;
return TSDB_CODE_SUCCESS;
}
int32_t convertJSONNumber(TAOS_SML_KV *pVal, char* typeStr, cJSON *value, SSmlLinesInfo* info) {
//tinyint
if (strcasecmp(typeStr, "i8") == 0 ||
strcasecmp(typeStr, "tinyint") == 0) {
if (!IS_VALID_TINYINT(value->valueint)) {
tscError("OTD:0x%"PRIx64" JSON value(%"PRId64") cannot fit in type(tinyint)", info->id, value->valueint);
return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE;
}
pVal->type = TSDB_DATA_TYPE_TINYINT;
pVal->length = (int16_t)tDataTypes[pVal->type].bytes;
pVal->value = tcalloc(pVal->length, 1);
*(int8_t *)(pVal->value) = (int8_t)(value->valueint);
return TSDB_CODE_SUCCESS;
}
//smallint
if (strcasecmp(typeStr, "i16") == 0 ||
strcasecmp(typeStr, "smallint") == 0) {
if (!IS_VALID_SMALLINT(value->valueint)) {
tscError("OTD:0x%"PRIx64" JSON value(%"PRId64") cannot fit in type(smallint)", info->id, value->valueint);
return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE;
}
pVal->type = TSDB_DATA_TYPE_SMALLINT;
pVal->length = (int16_t)tDataTypes[pVal->type].bytes;
pVal->value = tcalloc(pVal->length, 1);
*(int16_t *)(pVal->value) = (int16_t)(value->valueint);
return TSDB_CODE_SUCCESS;
}
//int
if (strcasecmp(typeStr, "i32") == 0 ||
strcasecmp(typeStr, "int") == 0) {
if (!IS_VALID_INT(value->valueint)) {
tscError("OTD:0x%"PRIx64" JSON value(%"PRId64") cannot fit in type(int)", info->id, value->valueint);
return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE;
}
pVal->type = TSDB_DATA_TYPE_INT;
pVal->length = (int16_t)tDataTypes[pVal->type].bytes;
pVal->value = tcalloc(pVal->length, 1);
*(int32_t *)(pVal->value) = (int32_t)(value->valueint);
return TSDB_CODE_SUCCESS;
}
//bigint
if (strcasecmp(typeStr, "i64") == 0 ||
strcasecmp(typeStr, "bigint") == 0) {
if (!IS_VALID_BIGINT(value->valueint)) {
tscError("OTD:0x%"PRIx64" JSON value(%"PRId64") cannot fit in type(bigint)", info->id, value->valueint);
return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE;
}
pVal->type = TSDB_DATA_TYPE_BIGINT;
pVal->length = (int16_t)tDataTypes[pVal->type].bytes;
pVal->value = tcalloc(pVal->length, 1);
*(int64_t *)(pVal->value) = (int64_t)(value->valueint);
return TSDB_CODE_SUCCESS;
}
//float
if (strcasecmp(typeStr, "f32") == 0 ||
strcasecmp(typeStr, "float") == 0) {
if (!IS_VALID_FLOAT(value->valuedouble)) {
tscError("OTD:0x%"PRIx64" JSON value(%f) cannot fit in type(float)", info->id, value->valuedouble);
return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE;
}
pVal->type = TSDB_DATA_TYPE_FLOAT;
pVal->length = (int16_t)tDataTypes[pVal->type].bytes;
pVal->value = tcalloc(pVal->length, 1);
*(float *)(pVal->value) = (float)(value->valuedouble);
return TSDB_CODE_SUCCESS;
}
//double
if (strcasecmp(typeStr, "f64") == 0 ||
strcasecmp(typeStr, "double") == 0) {
if (!IS_VALID_DOUBLE(value->valuedouble)) {
tscError("OTD:0x%"PRIx64" JSON value(%f) cannot fit in type(double)", info->id, value->valuedouble);
return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE;
}
pVal->type = TSDB_DATA_TYPE_DOUBLE;
pVal->length = (int16_t)tDataTypes[pVal->type].bytes;
pVal->value = tcalloc(pVal->length, 1);
*(double *)(pVal->value) = (double)(value->valuedouble);
return TSDB_CODE_SUCCESS;
}
//if reach here means type is unsupported
tscError("OTD:0x%"PRIx64" invalid type(%s) for JSON Number", info->id, typeStr);
return TSDB_CODE_TSC_INVALID_JSON_TYPE;
}
int32_t convertJSONString(TAOS_SML_KV *pVal, char* typeStr, cJSON *value, SSmlLinesInfo* info) {
if (strcasecmp(typeStr, "binary") == 0) {
pVal->type = TSDB_DATA_TYPE_BINARY;
} else if (strcasecmp(typeStr, "nchar") == 0) {
pVal->type = TSDB_DATA_TYPE_NCHAR;
} else {
tscError("OTD:0x%"PRIx64" invalid type(%s) for JSON String", info->id, typeStr);
return TSDB_CODE_TSC_INVALID_JSON_TYPE;
}
pVal->length = (int16_t)strlen(value->valuestring);
pVal->value = tcalloc(pVal->length + 1, 1);
memcpy(pVal->value, value->valuestring, pVal->length);
return TSDB_CODE_SUCCESS;
}
int32_t parseValueFromJSONObj(cJSON *root, TAOS_SML_KV *pVal, SSmlLinesInfo* info) {
int32_t ret = TSDB_CODE_SUCCESS;
int32_t size = cJSON_GetArraySize(root);
if (size != OTD_JSON_SUB_FIELDS_NUM) {
return TSDB_CODE_TSC_INVALID_JSON;
}
cJSON *value = cJSON_GetObjectItem(root, "value");
if (value == NULL) {
return TSDB_CODE_TSC_INVALID_JSON;
}
cJSON *type = cJSON_GetObjectItem(root, "type");
if (!cJSON_IsString(type)) {
return TSDB_CODE_TSC_INVALID_JSON;
}
switch (value->type) {
case cJSON_True:
case cJSON_False: {
ret = convertJSONBool(pVal, type->valuestring, value->valueint, info);
if (ret != TSDB_CODE_SUCCESS) {
return ret;
}
break;
}
case cJSON_Number: {
ret = convertJSONNumber(pVal, type->valuestring, value, info);
if (ret != TSDB_CODE_SUCCESS) {
return ret;
}
break;
}
case cJSON_String: {
ret = convertJSONString(pVal, type->valuestring, value, info);
if (ret != TSDB_CODE_SUCCESS) {
return ret;
}
break;
}
default:
return TSDB_CODE_TSC_INVALID_JSON_TYPE;
}
return TSDB_CODE_SUCCESS;
}
int32_t parseValueFromJSON(cJSON *root, TAOS_SML_KV *pVal, SSmlLinesInfo* info) {
int type = root->type;
switch (type) {
case cJSON_True:
case cJSON_False: {
pVal->type = TSDB_DATA_TYPE_BOOL;
pVal->length = (int16_t)tDataTypes[pVal->type].bytes;
pVal->value = tcalloc(pVal->length, 1);
*(bool *)(pVal->value) = root->valueint ? true : false;
break;
}
case cJSON_Number: {
//convert default JSON Number type to float
pVal->type = TSDB_DATA_TYPE_FLOAT;
pVal->length = (int16_t)tDataTypes[pVal->type].bytes;
pVal->value = tcalloc(pVal->length, 1);
*(float *)(pVal->value) = (float)(root->valuedouble);
break;
}
case cJSON_String: {
//convert default JSON String type to nchar
pVal->type = TSDB_DATA_TYPE_NCHAR;
//pVal->length = wcslen((wchar_t *)root->valuestring) * TSDB_NCHAR_SIZE;
pVal->length = (int16_t)strlen(root->valuestring);
pVal->value = tcalloc(pVal->length + 1, 1);
memcpy(pVal->value, root->valuestring, pVal->length);
break;
}
case cJSON_Object: {
int32_t ret = parseValueFromJSONObj(root, pVal, info);
if (ret != TSDB_CODE_SUCCESS) {
tscError("OTD:0x%"PRIx64" Failed to parse timestamp from JSON Obj", info->id);
return ret;
}
break;
}
default:
return TSDB_CODE_TSC_INVALID_JSON;
}
return TSDB_CODE_SUCCESS;
}
int32_t parseMetricValueFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, SSmlLinesInfo* info) {
//skip timestamp
TAOS_SML_KV *pVal = *pKVs + 1;
char key[] = OTD_METRIC_VALUE_COLUMN_NAME;
cJSON *metricVal = cJSON_GetObjectItem(root, "value");
if (metricVal == NULL) {
return TSDB_CODE_TSC_INVALID_JSON;
}
int32_t ret = parseValueFromJSON(metricVal, pVal, info);
if (ret != TSDB_CODE_SUCCESS) {
return ret;
}
pVal->key = tcalloc(sizeof(key), 1);
memcpy(pVal->key, key, sizeof(key));
*num_kvs += 1;
return TSDB_CODE_SUCCESS;
}
int32_t parseTagsFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, char **childTableName, SSmlLinesInfo* info) {
int32_t ret = TSDB_CODE_SUCCESS;
cJSON *tags = cJSON_GetObjectItem(root, "tags");
if (tags == NULL || tags->type != cJSON_Object) {
return TSDB_CODE_TSC_INVALID_JSON;
}
//only pick up the first ID value as child table name
cJSON *id = cJSON_GetObjectItem(tags, "ID");
if (id != NULL) {
size_t idLen = strlen(id->valuestring);
ret = isValidChildTableName(id->valuestring, (int16_t)idLen);
if (ret != TSDB_CODE_SUCCESS) {
return ret;
}
*childTableName = tcalloc(idLen + 1, sizeof(char));
memcpy(*childTableName, id->valuestring, idLen);
//remove all ID fields from tags list no case sensitive
while (id != NULL) {
cJSON_DeleteItemFromObject(tags, "ID");
id = cJSON_GetObjectItem(tags, "ID");
}
}
int32_t tagNum = cJSON_GetArraySize(tags);
//at least one tag pair required
if (tagNum <= 0) {
return TSDB_CODE_TSC_INVALID_JSON;
}
//allocate memory for tags
*pKVs = tcalloc(tagNum, sizeof(TAOS_SML_KV));
TAOS_SML_KV *pkv = *pKVs;
for (int32_t i = 0; i < tagNum; ++i) {
cJSON *tag = cJSON_GetArrayItem(tags, i);
if (tag == NULL) {
return TSDB_CODE_TSC_INVALID_JSON;
}
//key
size_t keyLen = strlen(tag->string);
pkv->key = tcalloc(keyLen + 1, sizeof(char));
strncpy(pkv->key, tag->string, keyLen);
//value
ret = parseValueFromJSON(tag, pkv, info);
if (ret != TSDB_CODE_SUCCESS) {
return ret;
}
*num_kvs += 1;
pkv++;
}
return ret;
}
int32_t tscParseJSONPayload(cJSON *root, TAOS_SML_DATA_POINT* pSml, SSmlLinesInfo* info) {
int32_t ret = TSDB_CODE_SUCCESS;
if (!cJSON_IsObject(root)) {
tscError("OTD:0x%"PRIx64" data point needs to be JSON object", info->id);
return TSDB_CODE_TSC_INVALID_JSON;
}
int32_t size = cJSON_GetArraySize(root);
//outmost json fields has to be exactly 4
if (size != OTD_JSON_FIELDS_NUM) {
tscError("OTD:0x%"PRIx64" Invalid number of JSON fields in data point %d", info->id, size);
return TSDB_CODE_TSC_INVALID_JSON;
}
//Parse metric
ret = parseMetricFromJSON(root, pSml, info);
if (ret != TSDB_CODE_SUCCESS) {
tscError("OTD:0x%"PRIx64" Unable to parse metric from JSON payload", info->id);
return ret;
}
tscDebug("OTD:0x%"PRIx64" Parse metric from JSON payload finished", info->id);
//Parse timestamp
ret = parseTimestampFromJSON(root, &pSml->fields, &pSml->fieldNum, info);
if (ret) {
tscError("OTD:0x%"PRIx64" Unable to parse timestamp from JSON payload", info->id);
return ret;
}
tscDebug("OTD:0x%"PRIx64" Parse timestamp from JSON payload finished", info->id);
//Parse metric value
ret = parseMetricValueFromJSON(root, &pSml->fields, &pSml->fieldNum, info);
if (ret) {
tscError("OTD:0x%"PRIx64" Unable to parse metric value from JSON payload", info->id);
return ret;
}
tscDebug("OTD:0x%"PRIx64" Parse metric value from JSON payload finished", info->id);
//Parse tags
ret = parseTagsFromJSON(root, &pSml->tags, &pSml->tagNum, &pSml->childTableName, info);
if (ret) {
tscError("OTD:0x%"PRIx64" Unable to parse tags from JSON payload", info->id);
return ret;
}
tscDebug("OTD:0x%"PRIx64" Parse tags from JSON payload finished", info->id);
return TSDB_CODE_SUCCESS;
}
int32_t tscParseMultiJSONPayload(char* payload, SArray* points, SSmlLinesInfo* info) {
int32_t payloadNum, ret;
ret = TSDB_CODE_SUCCESS;
if (payload == NULL) {
tscError("OTD:0x%"PRIx64" empty JSON Payload", info->id);
return TSDB_CODE_TSC_INVALID_JSON;
}
cJSON *root = cJSON_Parse(payload);
//multiple data points must be sent in JSON array
if (cJSON_IsObject(root)) {
payloadNum = 1;
} else if (cJSON_IsArray(root)) {
payloadNum = cJSON_GetArraySize(root);
} else {
tscError("OTD:0x%"PRIx64" Invalid JSON Payload", info->id);
ret = TSDB_CODE_TSC_INVALID_JSON;
goto PARSE_JSON_OVER;
}
for (int32_t i = 0; i < payloadNum; ++i) {
TAOS_SML_DATA_POINT point = {0};
cJSON *dataPoint = (payloadNum == 1) ? root : cJSON_GetArrayItem(root, i);
ret = tscParseJSONPayload(dataPoint, &point, info);
if (ret != TSDB_CODE_SUCCESS) {
tscError("OTD:0x%"PRIx64" JSON data point parse failed", info->id);
destroySmlDataPoint(&point);
goto PARSE_JSON_OVER;
} else {
tscDebug("OTD:0x%"PRIx64" JSON data point parse success", info->id);
}
taosArrayPush(points, &point);
}
PARSE_JSON_OVER:
cJSON_Delete(root);
return ret;
}
int taos_insert_json_payload(TAOS* taos, char* payload) {
int32_t code = 0;
SSmlLinesInfo* info = tcalloc(1, sizeof(SSmlLinesInfo));
info->id = genUID();
if (payload == NULL) {
tscError("OTD:0x%"PRIx64" taos_insert_json_payload payload is NULL", info->id);
tfree(info);
code = TSDB_CODE_TSC_APP_ERROR;
return code;
}
SArray* lpPoints = taosArrayInit(1, sizeof(TAOS_SML_DATA_POINT));
if (lpPoints == NULL) {
tscError("OTD:0x%"PRIx64" taos_insert_json_payload failed to allocate memory", info->id);
tfree(info);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
tscDebug("OTD:0x%"PRIx64" taos_insert_telnet_lines begin inserting %d points", info->id, 1);
code = tscParseMultiJSONPayload(payload, lpPoints, info);
size_t numPoints = taosArrayGetSize(lpPoints);
if (code != 0) {
goto cleanup;
}
TAOS_SML_DATA_POINT* points = TARRAY_GET_START(lpPoints);
code = tscSmlInsert(taos, points, (int)numPoints, info);
if (code != 0) {
tscError("OTD:0x%"PRIx64" taos_insert_json_payload error: %s", info->id, tstrerror((code)));
}
cleanup:
tscDebug("OTD:0x%"PRIx64" taos_insert_json_payload finish inserting 1 Point. code: %d", info->id, code);
points = TARRAY_GET_START(lpPoints);
numPoints = taosArrayGetSize(lpPoints);
for (int i = 0; i < numPoints; ++i) {
destroySmlDataPoint(points+i);
}
taosArrayDestroy(lpPoints);
tfree(info);
return code;
}

2075
2.0/src/client/src/tscPrepare.c
View File

File diff suppressed because it is too large Load Diff

368
2.0/src/client/src/tscProfile.c
View File

@ -1,368 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "tscLog.h"
#include "tsclient.h"
#include "tsocket.h"
#include "ttimer.h"
#include "tmsg.h"
#include "tcq.h"
#include "../../../include/client/taos.h"
#include "tscUtil.h"
void tscSaveSlowQueryFp(void *handle, void *tmrId);
TAOS *tscSlowQueryConn = NULL;
bool tscSlowQueryConnInitialized = false;
void tscInitConnCb(void *param, TAOS_RES *result, int code) {
char *sql = param;
if (code < 0) {
tscError("taos:%p, slow query connect failed, code:%d", tscSlowQueryConn, code);
taos_close(tscSlowQueryConn);
tscSlowQueryConn = NULL;
tscSlowQueryConnInitialized = false;
free(sql);
} else {
tscDebug("taos:%p, slow query connect success, code:%d", tscSlowQueryConn, code);
tscSlowQueryConnInitialized = true;
tscSaveSlowQueryFp(sql, NULL);
}
taos_free_result(result);
}
void tscAddIntoSqlList(SSqlObj *pSql) {
static uint32_t queryId = 1;
STscObj *pObj = pSql->pTscObj;
if (pSql->listed) return;
pthread_mutex_lock(&pObj->mutex);
assert(pSql != pObj->sqlList);
pSql->next = pObj->sqlList;
if (pObj->sqlList) pObj->sqlList->prev = pSql;
pObj->sqlList = pSql;
pSql->queryId = atomic_fetch_add_32(&queryId, 1);
pthread_mutex_unlock(&pObj->mutex);
pSql->stime = taosGetTimestampMs();
pSql->listed = 1;
tscDebug("0x%"PRIx64" added into sqlList, queryId:%u", pSql->self, pSql->queryId);
}
void tscSaveSlowQueryFpCb(void *param, TAOS_RES *result, int code) {
if (code < 0) {
tscError("failed to save slow query, code:%d", code);
} else {
tscDebug("success to save slow query, code:%d", code);
}
taos_free_result(result);
}
void tscSaveSlowQueryFp(void *handle, void *tmrId) {
char *sql = handle;
if (!tscSlowQueryConnInitialized) {
if (tscSlowQueryConn == NULL) {
tscDebug("start to init slow query connect");
taos_connect_a(NULL, "monitor", tsInternalPass, "", 0, tscInitConnCb, sql, &tscSlowQueryConn);
} else {
tscError("taos:%p, slow query connect is already initialized", tscSlowQueryConn);
free(sql);
}
} else {
tscDebug("taos:%p, save slow query:%s", tscSlowQueryConn, sql);
taos_query_a(tscSlowQueryConn, sql, tscSaveSlowQueryFpCb, NULL);
free(sql);
}
}
void tscSaveSlowQuery(SSqlObj *pSql) {
const static int64_t SLOW_QUERY_INTERVAL = 3000000L; // todo configurable
size_t size = 200; // other part of sql string, expect the main sql str
if (pSql->res.useconds < SLOW_QUERY_INTERVAL) {
return;
}
tscDebug("0x%"PRIx64" query time:%" PRId64 " sql:%s", pSql->self, pSql->res.useconds, pSql->sqlstr);
int32_t sqlSize = (int32_t)(TSDB_SLOW_QUERY_SQL_LEN + size);
char *sql = malloc(sqlSize);
if (sql == NULL) {
tscError("0x%"PRIx64" failed to allocate memory to sent slow query to dnode", pSql->self);
return;
}
int len = snprintf(sql, size, "insert into %s.slowquery values(now, '%s', %" PRId64 ", %" PRId64 ", '", tsMonitorDbName,
pSql->pTscObj->user, pSql->stime, pSql->res.useconds);
int sqlLen = snprintf(sql + len, TSDB_SLOW_QUERY_SQL_LEN, "%s", pSql->sqlstr);
if (sqlLen > TSDB_SLOW_QUERY_SQL_LEN - 1) {
sqlLen = len + TSDB_SLOW_QUERY_SQL_LEN - 1;
} else {
sqlLen += len;
}
strcpy(sql + sqlLen, "')");
taosTmrStart(tscSaveSlowQueryFp, 200, sql, tscTmr);
}
void tscRemoveFromSqlList(SSqlObj *pSql) {
STscObj *pObj = pSql->pTscObj;
if (pSql->listed == 0) return;
pthread_mutex_lock(&pObj->mutex);
if (pSql->prev)
pSql->prev->next = pSql->next;
else
pObj->sqlList = pSql->next;
if (pSql->next) pSql->next->prev = pSql->prev;
pthread_mutex_unlock(&pObj->mutex);
pSql->next = NULL;
pSql->prev = NULL;
pSql->listed = 0;
tscSaveSlowQuery(pSql);
tscDebug("0x%"PRIx64" removed from sqlList", pSql->self);
}
void tscKillQuery(STscObj *pObj, uint32_t killId) {
pthread_mutex_lock(&pObj->mutex);
SSqlObj *pSql = pObj->sqlList;
while (pSql) {
if (pSql->queryId == killId) break;
pSql = pSql->next;
}
pthread_mutex_unlock(&pObj->mutex);
if (pSql == NULL) {
tscError("failed to kill query, id:%d, it may have completed/terminated", killId);
} else {
tscDebug("0x%"PRIx64" query is killed, queryId:%d", pSql->self, killId);
taos_stop_query(pSql);
}
}
void tscAddIntoStreamList(SSqlStream *pStream) {
static uint32_t streamId = 1;
STscObj * pObj = pStream->pSql->pTscObj;
pthread_mutex_lock(&pObj->mutex);
pStream->next = pObj->streamList;
if (pObj->streamList) pObj->streamList->prev = pStream;
pObj->streamList = pStream;
pStream->streamId = streamId++;
pthread_mutex_unlock(&pObj->mutex);
pStream->listed = 1;
}
void tscRemoveFromStreamList(SSqlStream *pStream, SSqlObj *pSqlObj) {
if (pStream->listed == 0) return;
STscObj *pObj = pSqlObj->pTscObj;
pthread_mutex_lock(&pObj->mutex);
if (pStream->prev)
pStream->prev->next = pStream->next;
else
pObj->streamList = pStream->next;
if (pStream->next) pStream->next->prev = pStream->prev;
pthread_mutex_unlock(&pObj->mutex);
pStream->next = NULL;
pStream->prev = NULL;
pStream->listed = 0;
}
void tscKillStream(STscObj *pObj, uint32_t killId) {
pthread_mutex_lock(&pObj->mutex);
SSqlStream *pStream = pObj->streamList;
while (pStream) {
if (pStream->streamId == killId) break;
pStream = pStream->next;
}
pthread_mutex_unlock(&pObj->mutex);
if (pStream) {
tscDebug("0x%"PRIx64" stream:%p is killed, streamId:%d", pStream->pSql->self, pStream, killId);
if (pStream->callback) {
pStream->callback(pStream->param);
}
taos_close_stream(pStream);
} else {
tscError("failed to kill stream, streamId:%d not exist", killId);
}
}
int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) {
SHeartBeatReq *pHeartbeat = pMsg;
int allocedQueriesNum = pHeartbeat->numOfQueries;
int allocedStreamsNum = pHeartbeat->numOfStreams;
pHeartbeat->numOfQueries = 0;
SQueryDesc *pQdesc = (SQueryDesc *)pHeartbeat->pData;
int64_t now = taosGetTimestampMs();
SSqlObj *pSql = pObj->sqlList;
while (pSql) {
/*
* avoid sqlobj may not be correctly removed from sql list
* e.g., forgetting to free the sql result may cause the sql object still in sql list
*/
if (pSql->sqlstr == NULL) {
pSql = pSql->next;
continue;
}
tstrncpy(pQdesc->sql, pSql->sqlstr, sizeof(pQdesc->sql));
pQdesc->stime = htobe64(pSql->stime);
pQdesc->queryId = htonl(pSql->queryId);
pQdesc->useconds = htobe64(now - pSql->stime);
pQdesc->qId = htobe64(pSql->res.qId);
pQdesc->sqlObjId = htobe64(pSql->self);
pQdesc->pid = pHeartbeat->pid;
pQdesc->numOfSub = pSql->subState.numOfSub;
// todo race condition
pQdesc->stableQuery = 0;
char *p = pQdesc->subSqlInfo;
int32_t remainLen = sizeof(pQdesc->subSqlInfo);
if (pQdesc->numOfSub == 0) {
snprintf(p, remainLen, "N/A");
} else {
// SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd);
// if (pQueryInfo != NULL) {
// pQdesc->stableQuery = (pQueryInfo->stableQuery)?1:0;
// } else {
// pQdesc->stableQuery = 0;
// }
if (pSql->pSubs != NULL && pSql->subState.states != NULL) {
for (int32_t i = 0; i < pQdesc->numOfSub; ++i) {
SSqlObj *psub = pSql->pSubs[i];
int64_t self = (psub != NULL)? psub->self : 0;
int32_t len = snprintf(p, remainLen, "[%d]0x%" PRIx64 "(%c) ", i, self, pSql->subState.states[i] ? 'C' : 'I');
if (len > remainLen) {
break;
}
remainLen -= len;
p += len;
}
}
}
pQdesc->numOfSub = htonl(pQdesc->numOfSub);
taosGetFqdn(pQdesc->fqdn);
pHeartbeat->numOfQueries++;
pQdesc++;
pSql = pSql->next;
if (pHeartbeat->numOfQueries >= allocedQueriesNum) {
break;
}
}
pHeartbeat->numOfStreams = 0;
SStreamDesc *pSdesc = (SStreamDesc *)pQdesc;
SSqlStream *pStream = pObj->streamList;
while (pStream) {
tstrncpy(pSdesc->sql, pStream->pSql->sqlstr, sizeof(pSdesc->sql));
if (pStream->dstTable == NULL) {
pSdesc->dstTable[0] = 0;
} else {
tstrncpy(pSdesc->dstTable, pStream->dstTable, sizeof(pSdesc->dstTable));
}
pSdesc->streamId = htonl(pStream->streamId);
pSdesc->num = htobe64(pStream->num);
pSdesc->useconds = htobe64(pStream->useconds);
pSdesc->stime = (pStream->stime == INT64_MIN) ? htobe64(pStream->stime) : htobe64(pStream->stime - pStream->interval.interval);
pSdesc->ctime = htobe64(pStream->ctime);
pSdesc->slidingTime = htobe64(pStream->interval.sliding);
pSdesc->interval = htobe64(pStream->interval.interval);
pHeartbeat->numOfStreams++;
pSdesc++;
pStream = pStream->next;
if (pHeartbeat->numOfStreams >= allocedStreamsNum) break;
}
int32_t msgLen = pHeartbeat->numOfQueries * sizeof(SQueryDesc) + pHeartbeat->numOfStreams * sizeof(SStreamDesc) +
sizeof(SHeartBeatReq);
pHeartbeat->connId = htonl(pObj->connId);
pHeartbeat->numOfQueries = htonl(pHeartbeat->numOfQueries);
pHeartbeat->numOfStreams = htonl(pHeartbeat->numOfStreams);
return msgLen;
}
// cqContext->dbconn is killed then call this callback
void cqConnKilledNotify(void* handle, void* conn) {
if (handle == NULL || conn == NULL){
return ;
}
SCqContext* pContext = (SCqContext*) handle;
if (pContext->dbConn == conn){
atomic_store_ptr(&(pContext->dbConn), NULL);
}
}
void tscKillConnection(STscObj *pObj) {
// get stream header by locked
pthread_mutex_lock(&pObj->mutex);
SSqlStream *pStream = pObj->streamList;
pthread_mutex_unlock(&pObj->mutex);
while (pStream) {
SSqlStream *tmp = pStream->next;
// set associate variant to NULL
cqConnKilledNotify(pStream->cqhandle, pObj);
// taos_close_stream function call pObj->mutet lock , careful death-lock
taos_close_stream(pStream);
pStream = tmp;
}
tscDebug("connection:%p is killed", pObj);
taos_close(pObj);
}

9411
2.0/src/client/src/tscSQLParser.c
View File

File diff suppressed because it is too large Load Diff

3222
2.0/src/client/src/tscServer.c
View File

File diff suppressed because it is too large Load Diff

1021
2.0/src/client/src/tscSql.c
View File

File diff suppressed because it is too large Load Diff

776
2.0/src/client/src/tscStream.c
View File

@ -1,776 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "tmsg.h"
#include "tscLog.h"
#include "tscUtil.h"
#include "tsched.h"
#include "tcache.h"
#include "tsclient.h"
#include "ttimer.h"
#include "tutil.h"
#include "tscProfile.h"
#include "tscSubquery.h"
static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOfRows);
static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOfRows);
static void tscSetNextLaunchTimer(SSqlStream *pStream, SSqlObj *pSql);
static void tscSetRetryTimer(SSqlStream *pStream, SSqlObj *pSql, int64_t timer);
static int64_t getDelayValueAfterTimewindowClosed(SSqlStream* pStream, int64_t launchDelay) {
return taosGetTimestamp(pStream->precision) + launchDelay - pStream->stime - 1;
}
static bool isProjectStream(SQueryInfo* pQueryInfo) {
for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
SExprInfo *pExpr = tscExprGet(pQueryInfo, i);
if (pExpr->base.functionId != TSDB_FUNC_PRJ) {
return false;
}
}
return true;
}
static int64_t tscGetRetryDelayTime(SSqlStream* pStream, int64_t slidingTime, int16_t prec) {
float retryRangeFactor = 0.3f;
int64_t retryDelta = (int64_t)(tsRetryStreamCompDelay * retryRangeFactor);
retryDelta = ((rand() % retryDelta) + tsRetryStreamCompDelay) * 1000L;
if (pStream->interval.intervalUnit != 'n' && pStream->interval.intervalUnit != 'y') {
// change to ms
slidingTime = convertTimePrecision(slidingTime, pStream->precision, TSDB_TIME_PRECISION_MILLI);
if (slidingTime < retryDelta) {
return slidingTime;
}
}
return retryDelta;
}
static void setRetryInfo(SSqlStream* pStream, int32_t code) {
SSqlObj* pSql = pStream->pSql;
pSql->res.code = code;
int64_t retryDelayTime = tscGetRetryDelayTime(pStream, pStream->interval.sliding, pStream->precision);
tscDebug("0x%"PRIx64" stream:%p, get table Meta failed, retry in %" PRId64 "ms", pSql->self, pStream, retryDelayTime);
tscSetRetryTimer(pStream, pSql, retryDelayTime);
}
static void doLaunchQuery(void* param, TAOS_RES* tres, int32_t code) {
SSqlStream *pStream = (SSqlStream *)param;
assert(pStream->pSql == tres);
SSqlObj* pSql = (SSqlObj*) tres;
pSql->fp = doLaunchQuery;
pSql->fetchFp = doLaunchQuery;
pSql->res.completed = false;
if (code != TSDB_CODE_SUCCESS) {
setRetryInfo(pStream, code);
return;
}
SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd);
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
code = tscGetTableMeta(pSql, pTableMetaInfo);
if (code == 0 && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
code = tscGetSTableVgroupInfo(pSql, pQueryInfo);
}
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
return;
}
if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo) && (pTableMetaInfo->pVgroupTables == NULL) && (pTableMetaInfo->vgroupList == NULL || pTableMetaInfo->vgroupList->numOfVgroups <= 0)) {
tscDebug("0x%"PRIx64" empty vgroup list", pSql->self);
pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList);
code = TSDB_CODE_TSC_APP_ERROR;
}
// failed to get table Meta or vgroup list, retry in 10sec.
if (code == TSDB_CODE_SUCCESS) {
tscTansformFuncForSTableQuery(pQueryInfo);
tscDebug("0x%"PRIx64" stream:%p, start stream query on:%s QueryInfo->skey=%"PRId64" ekey=%"PRId64" ", pSql->self, pStream, tNameGetTableName(&pTableMetaInfo->name), pQueryInfo->window.skey, pQueryInfo->window.ekey);
pQueryInfo->command = TSDB_SQL_SELECT;
pSql->fp = tscProcessStreamQueryCallback;
pSql->fetchFp = tscProcessStreamQueryCallback;
executeQuery(pSql, pQueryInfo);
tscIncStreamExecutionCount(pStream);
} else {
setRetryInfo(pStream, code);
}
}
static void tscProcessStreamLaunchQuery(SSchedMsg *pMsg) {
SSqlStream *pStream = (SSqlStream *)pMsg->ahandle;
doLaunchQuery(pStream, pStream->pSql, 0);
}
static void tscProcessStreamTimer(void *handle, void *tmrId) {
SSqlStream *pStream = (SSqlStream *)handle;
if (pStream == NULL || pStream->pTimer != tmrId) {
return;
}
pStream->pTimer = NULL;
pStream->numOfRes = 0; // reset the numOfRes.
SSqlObj *pSql = pStream->pSql;
// pSql == NULL maybe killStream already called
if(pSql == NULL) {
return ;
}
SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd);
tscDebug("0x%"PRIx64" add into timer", pSql->self);
if (pStream->isProject) {
/*
* pQueryInfo->window.ekey, which is the start time, does not change in case of
* repeat first execution, once the first execution failed.
*/
pQueryInfo->window.skey = pStream->stime; // start time
pQueryInfo->window.ekey = taosGetTimestamp(pStream->precision); // end time
if (pQueryInfo->window.ekey > pStream->etime) {
pQueryInfo->window.ekey = pStream->etime;
}
} else {
pQueryInfo->window.skey = pStream->stime;
int64_t etime = taosGetTimestamp(pStream->precision);
// delay to wait all data in last time window
etime -= convertTimePrecision(tsMaxStreamComputDelay, TSDB_TIME_PRECISION_MILLI, pStream->precision);
if (etime > pStream->etime) {
etime = pStream->etime;
} else if (pStream->interval.intervalUnit != 'y' && pStream->interval.intervalUnit != 'n') {
if(pStream->stime == INT64_MIN) {
etime = taosTimeTruncate(etime, &pStream->interval, pStream->precision);
} else {
etime = pStream->stime + (etime - pStream->stime) / pStream->interval.interval * pStream->interval.interval;
}
} else {
etime = taosTimeTruncate(etime, &pStream->interval, pStream->precision);
}
pQueryInfo->window.ekey = etime;
if (pQueryInfo->window.skey >= pQueryInfo->window.ekey) {
int64_t timer = pStream->interval.sliding;
if (pStream->interval.intervalUnit == 'y' || pStream->interval.intervalUnit == 'n') {
timer = 86400 * 1000l;
} else {
timer = convertTimePrecision(timer, pStream->precision, TSDB_TIME_PRECISION_MILLI);
}
tscSetRetryTimer(pStream, pSql, timer);
return;
}
}
// launch stream computing in a new thread
SSchedMsg schedMsg = {0};
schedMsg.fp = tscProcessStreamLaunchQuery;
schedMsg.ahandle = pStream;
schedMsg.thandle = (void *)1;
schedMsg.msg = NULL;
taosScheduleTask(tscQhandle, &schedMsg);
}
static void cbParseSql(void* param, TAOS_RES* res, int code);
static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOfRows) {
SSqlStream *pStream = (SSqlStream *)param;
if (tres == NULL || numOfRows < 0) {
int64_t retryDelay = tscGetRetryDelayTime(pStream, pStream->interval.sliding, pStream->precision);
tscError("0x%"PRIx64" stream:%p, query data failed, code:0x%08x, retry in %" PRId64 "ms", pStream->pSql->self,
pStream, numOfRows, retryDelay);
SSqlObj* pSql = pStream->pSql;
tscFreeSqlResult(pSql);
tscFreeSubobj(pSql);
tfree(pSql->pSubs);
pSql->subState.numOfSub = 0;
int32_t code = tsParseSql(pSql, true);
if (code == TSDB_CODE_SUCCESS) {
cbParseSql(pStream, pSql, code);
} else if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
tscDebug("0x%"PRIx64" CQ taso_open_stream IN Process", pSql->self);
} else {
tscError("0x%"PRIx64" open stream failed, code:%s", pSql->self, tstrerror(code));
taosReleaseRef(tscObjRef, pSql->self);
free(pStream);
}
// tscSetRetryTimer(pStream, pStream->pSql, retryDelay);
// return;
}
taos_fetch_rows_a(tres, tscProcessStreamRetrieveResult, param);
}
// no need to be called as this is alreay done in the query
static void tscStreamFillTimeGap(SSqlStream* pStream, TSKEY ts) {
#if 0
SSqlObj * pSql = pStream->pSql;
SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd);
if (pQueryInfo->fillType != TSDB_FILL_SET_VALUE && pQueryInfo->fillType != TSDB_FILL_NULL) {
return;
}
SSqlRes *pRes = &pSql->res;
/* failed to retrieve any result in this retrieve */
pSql->res.numOfRows = 1;
void *row[TSDB_MAX_COLUMNS] = {0};
char tmpRes[TSDB_MAX_BYTES_PER_ROW] = {0};
void *oldPtr = pSql->res.data;
pSql->res.data = tmpRes;
int32_t rowNum = 0;
while (pStream->stime + pStream->slidingTime < ts) {
pStream->stime += pStream->slidingTime;
*(TSKEY*)row[0] = pStream->stime;
for (int32_t i = 1; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
int16_t offset = tscFieldInfoGetOffset(pQueryInfo, i);
TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
assignVal(pSql->res.data + offset, (char *)(&pQueryInfo->fillVal[i]), pField->bytes, pField->type);
row[i] = pSql->res.data + offset;
}
(*pStream->fp)(pStream->param, pSql, row);
++rowNum;
}
if (rowNum > 0) {
tscDebug("0x%"PRIx64" stream:%p %d rows padded", pSql, pStream, rowNum);
}
pRes->numOfRows = 0;
pRes->data = oldPtr;
#endif
}
static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOfRows) {
SSqlStream * pStream = (SSqlStream *)param;
SSqlObj * pSql = (SSqlObj *)res;
if (pSql == NULL || numOfRows < 0) {
int64_t retryDelayTime = tscGetRetryDelayTime(pStream, pStream->interval.sliding, pStream->precision);
tscError("stream:%p, retrieve data failed, code:0x%08x, retry in %" PRId64 " ms", pStream, numOfRows, retryDelayTime);
tscSetRetryTimer(pStream, pStream->pSql, retryDelayTime);
return;
}
SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd);
STableMetaInfo *pTableMetaInfo = pQueryInfo->pTableMetaInfo[0];
if (numOfRows > 0) { // when reaching here the first execution of stream computing is successful.
for(int32_t i = 0; i < numOfRows; ++i) {
TAOS_ROW row = taos_fetch_row(res);
if (row != NULL) {
tscDebug("0x%"PRIx64" stream:%p fetch result", pSql->self, pStream);
tscStreamFillTimeGap(pStream, *(TSKEY*)row[0]);
pStream->stime = *(TSKEY *)row[0];
// user callback function
(*pStream->fp)(pStream->param, res, row);
pStream->numOfRes++;
}
}
if (!pStream->isProject) {
pStream->stime = taosTimeAdd(pStream->stime, pStream->interval.sliding, pStream->interval.slidingUnit, pStream->precision);
}
// actually only one row is returned. this following is not necessary
taos_fetch_rows_a(res, tscProcessStreamRetrieveResult, pStream);
} else { // numOfRows == 0, all data has been retrieved
pStream->useconds += pSql->res.useconds;
if (pStream->numOfRes == 0) {
if (pStream->isProject) {
/* no resuls in the query range, retry */
// todo set retry dynamic time
int32_t retry = tsProjectExecInterval;
tscError("0x%"PRIx64" stream:%p, retrieve no data, code:0x%08x, retry in %" PRId32 "ms", pSql->self, pStream, numOfRows, retry);
tscSetRetryTimer(pStream, pStream->pSql, retry);
return;
}
} else if (pStream->isProject) {
pStream->stime += 1;
}
tscDebug("0x%"PRIx64" stream:%p, query on:%s, fetch result completed, fetched rows:%" PRId64, pSql->self, pStream, tNameGetTableName(&pTableMetaInfo->name),
pStream->numOfRes);
tfree(pTableMetaInfo->pTableMeta);
if (pQueryInfo->pQInfo != NULL) {
qDestroyQueryInfo(pQueryInfo->pQInfo);
pQueryInfo->pQInfo = NULL;
}
tscFreeSqlResult(pSql);
tscFreeSubobj(pSql);
tfree(pSql->pSubs);
pSql->subState.numOfSub = 0;
pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList);
tscSetNextLaunchTimer(pStream, pSql);
}
}
static void tscSetRetryTimer(SSqlStream *pStream, SSqlObj *pSql, int64_t timer) {
int64_t delay = getDelayValueAfterTimewindowClosed(pStream, timer);
if (pStream->isProject) {
int64_t now = taosGetTimestamp(pStream->precision);
int64_t etime = now > pStream->etime ? pStream->etime : now;
int64_t maxRetent = tsMaxRetentWindow * 1000;
if(pStream->precision == TSDB_TIME_PRECISION_MICRO) {
maxRetent *= 1000;
}
if (pStream->etime < now && now - pStream->etime > maxRetent) {
/*
* current time window will be closed, since it too early to exceed the maxRetentWindow value
*/
tscDebug("0x%"PRIx64" stream:%p, etime:%" PRId64 " is too old, exceeds the max retention time window:%" PRId64 ", stop the stream",
pStream->pSql->self, pStream, pStream->stime, pStream->etime);
// TODO : How to terminate stream here
if (pStream->callback) {
// Callback function from upper level
pStream->callback(pStream->param);
}
taos_close_stream(pStream);
return;
}
tscDebug("0x%"PRIx64" stream:%p, next start at %" PRId64 "(ts window ekey), in %" PRId64 " ms. delay:%" PRId64 "ms qrange %" PRId64 "-%" PRId64, pStream->pSql->self, pStream,
now + timer, timer, delay, pStream->stime, etime);
} else {
tscDebug("0x%"PRIx64" stream:%p, next start at %" PRId64 " - %" PRId64 " end, in %" PRId64 "ms. delay:%" PRId64 "ms qrange %" PRId64 "-%" PRId64, pStream->pSql->self, pStream,
pStream->stime, pStream->etime, timer, delay, pStream->stime - pStream->interval.interval, pStream->stime - 1);
}
pSql->cmd.command = TSDB_SQL_SELECT;
// start timer for next computing
taosTmrReset(tscProcessStreamTimer, (int32_t)timer, pStream, tscTmr, &pStream->pTimer);
}
static int64_t getLaunchTimeDelay(const SSqlStream* pStream) {
int64_t maxDelay = convertTimePrecision(tsMaxStreamComputDelay, TSDB_TIME_PRECISION_MILLI, pStream->precision);
int64_t delayDelta = maxDelay;
if (pStream->interval.intervalUnit != 'n' && pStream->interval.intervalUnit != 'y') {
delayDelta = (int64_t)(pStream->interval.sliding * tsStreamComputDelayRatio);
if (delayDelta > maxDelay) {
delayDelta = maxDelay;
}
int64_t remainTimeWindow = pStream->interval.sliding - delayDelta;
if (maxDelay > remainTimeWindow) {
maxDelay = (int64_t)(remainTimeWindow / 1.5f);
}
}
int64_t currentDelay = (rand() % maxDelay); // a random number
currentDelay += delayDelta;
if (pStream->interval.intervalUnit != 'n' && pStream->interval.intervalUnit != 'y') {
assert(currentDelay < pStream->interval.sliding);
}
return currentDelay;
}
static void tscSetNextLaunchTimer(SSqlStream *pStream, SSqlObj *pSql) {
int64_t timer = 0;
if (pStream->isProject) {
/*
* for project query, no mater fetch data successfully or not, next launch will issue
* more than the sliding time window
*/
timer = pStream->interval.sliding;
if (pStream->stime > pStream->etime) {
tscDebug("0x%"PRIx64" stream:%p, stime:%" PRId64 " is larger than end time: %" PRId64 ", stop the stream",
pStream->pSql->self, pStream, pStream->stime, pStream->etime);
// TODO : How to terminate stream here
if (pStream->callback) {
// Callback function from upper level
pStream->callback(pStream->param);
}
taos_close_stream(pStream);
return;
}
} else {
int64_t stime = taosTimeTruncate(pStream->stime - 1, &pStream->interval, pStream->precision);
if (stime >= pStream->etime) {
tscDebug("0x%"PRIx64" stream:%p, stime:%" PRId64 " is larger than end time: %" PRId64 ", stop the stream", pStream->pSql->self, pStream,
pStream->stime, pStream->etime);
// TODO : How to terminate stream here
if (pStream->callback) {
// Callback function from upper level
pStream->callback(pStream->param);
}
taos_close_stream(pStream);
return;
}
if (pStream->stime > 0) {
timer = pStream->stime - taosGetTimestamp(pStream->precision);
if (timer < 0) {
timer = 0;
}
}
}
timer += getLaunchTimeDelay(pStream);
timer = convertTimePrecision(timer, pStream->precision, TSDB_TIME_PRECISION_MILLI);
tscSetRetryTimer(pStream, pSql, timer);
}
static int32_t tscSetSlidingWindowInfo(SSqlObj *pSql, SSqlStream *pStream) {
int64_t minIntervalTime =
convertTimePrecision(tsMinIntervalTime, TSDB_TIME_PRECISION_MILLI, pStream->precision);
SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd);
if (!pStream->isProject && pQueryInfo->interval.interval == 0) {
sprintf(pSql->cmd.payload, "the interval value is 0");
return -1;
}
if (pQueryInfo->interval.intervalUnit != 'n' && pQueryInfo->interval.intervalUnit!= 'y' && pQueryInfo->interval.interval < minIntervalTime) {
tscWarn("0x%"PRIx64" stream:%p, original sample interval:%" PRId64 " too small, reset to:%" PRId64, pSql->self, pStream,
(int64_t)pQueryInfo->interval.interval, minIntervalTime);
pQueryInfo->interval.interval = minIntervalTime;
}
pStream->interval.intervalUnit = pQueryInfo->interval.intervalUnit;
pStream->interval.interval = pQueryInfo->interval.interval; // it shall be derived from sql string
if (pQueryInfo->interval.sliding <= 0) {
pQueryInfo->interval.sliding = pQueryInfo->interval.interval;
pQueryInfo->interval.slidingUnit = pQueryInfo->interval.intervalUnit;
}
int64_t minSlidingTime =
convertTimePrecision(tsMinSlidingTime, TSDB_TIME_PRECISION_MILLI, pStream->precision);
if (pQueryInfo->interval.intervalUnit != 'n' && pQueryInfo->interval.intervalUnit!= 'y' && pQueryInfo->interval.sliding < minSlidingTime) {
tscWarn("0x%"PRIx64" stream:%p, original sliding value:%" PRId64 " too small, reset to:%" PRId64, pSql->self, pStream,
pQueryInfo->interval.sliding, minSlidingTime);
pQueryInfo->interval.sliding = minSlidingTime;
}
if (pQueryInfo->interval.sliding > pQueryInfo->interval.interval) {
tscWarn("0x%"PRIx64" stream:%p, sliding value:%" PRId64 " can not be larger than interval range, reset to:%" PRId64, pSql->self, pStream,
pQueryInfo->interval.sliding, pQueryInfo->interval.interval);
pQueryInfo->interval.sliding = pQueryInfo->interval.interval;
}
pStream->interval.slidingUnit = pQueryInfo->interval.slidingUnit;
pStream->interval.sliding = pQueryInfo->interval.sliding;
if (pStream->isProject) {
pQueryInfo->interval.interval = 0; // clear the interval value to avoid the force time window split by query processor
pQueryInfo->interval.sliding = 0;
}
return TSDB_CODE_SUCCESS;
}
static int64_t tscGetStreamStartTimestamp(SSqlObj *pSql, SSqlStream *pStream, int64_t stime) {
SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd);
if (pStream->isProject) {
// no data in table, flush all data till now to destination meter, 10sec delay
pStream->interval.interval = tsProjectExecInterval;
pStream->interval.sliding = tsProjectExecInterval;
if (stime != INT64_MIN) { // first projection start from the latest event timestamp
assert(stime >= pQueryInfo->window.skey);
stime += 1; // exclude the last records from table
} else {
stime = pQueryInfo->window.skey;
}
} else { // timewindow based aggregation stream
if (stime == INT64_MIN) { // no data in meter till now
if (pQueryInfo->window.skey != INT64_MIN) {
stime = pQueryInfo->window.skey;
} else {
return stime;
}
stime = taosTimeTruncate(stime, &pStream->interval, pStream->precision);
} else {
int64_t newStime = taosTimeTruncate(stime, &pStream->interval, pStream->precision);
if (newStime != stime) {
tscWarn("0x%"PRIx64" stream:%p, last timestamp:%" PRId64 ", reset to:%" PRId64, pSql->self, pStream, stime, newStime);
stime = newStime;
}
}
}
return stime;
}
static int64_t tscGetLaunchTimestamp(const SSqlStream *pStream) {
int64_t timer = 0, now = taosGetTimestamp(pStream->precision);
if (pStream->stime > now) {
timer = pStream->stime - now;
}
int64_t startDelay = convertTimePrecision(tsStreamCompStartDelay, TSDB_TIME_PRECISION_MILLI, pStream->precision);
timer += getLaunchTimeDelay(pStream);
timer += startDelay;
return convertTimePrecision(timer, pStream->precision, TSDB_TIME_PRECISION_MILLI);
}
static void tscCreateStream(void *param, TAOS_RES *res, int code) {
SSqlStream* pStream = (SSqlStream*)param;
SSqlObj* pSql = pStream->pSql;
SSqlCmd* pCmd = &pSql->cmd;
if (code != TSDB_CODE_SUCCESS) {
pSql->res.code = code;
tscError("0x%"PRIx64" open stream failed, sql:%s, reason:%s, code:%s", pSql->self, pSql->sqlstr, pCmd->payload, tstrerror(code));
pStream->fp(pStream->param, NULL, NULL);
return;
}
SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd);
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
pStream->isProject = isProjectStream(pQueryInfo);
pStream->precision = tinfo.precision;
pStream->ctime = taosGetTimestamp(pStream->precision);
pStream->etime = pQueryInfo->window.ekey;
if (tscSetSlidingWindowInfo(pSql, pStream) != TSDB_CODE_SUCCESS) {
pSql->res.code = code;
tscError("0x%"PRIx64" stream %p open failed, since the interval value is incorrect", pSql->self, pStream);
pStream->fp(pStream->param, NULL, NULL);
return;
}
pStream->stime = tscGetStreamStartTimestamp(pSql, pStream, pStream->stime);
// set stime with ltime if ltime > stime
const char* dstTable = pStream->dstTable? pStream->dstTable: "";
tscDebug("0x%"PRIx64" CQ table %s ltime is %"PRId64, pSql->self, dstTable, pStream->ltime);
if(pStream->ltime != INT64_MIN && pStream->ltime > pStream->stime) {
tscWarn("0x%"PRIx64" CQ set stream %s stime=%"PRId64" replace with ltime=%"PRId64" if ltime > 0", pSql->self, dstTable, pStream->stime, pStream->ltime);
pStream->stime = pStream->ltime;
}
int64_t starttime = tscGetLaunchTimestamp(pStream);
pCmd->command = TSDB_SQL_SELECT;
tscAddIntoStreamList(pStream);
taosTmrReset(tscProcessStreamTimer, (int32_t)starttime, pStream, tscTmr, &pStream->pTimer);
tscDebug("0x%"PRIx64" stream:%p is opened, query on:%s, interval:%" PRId64 ", sliding:%" PRId64 ", first launched in:%" PRId64 ", sql:%s", pSql->self,
pStream, tNameGetTableName(&pTableMetaInfo->name), pStream->interval.interval, pStream->interval.sliding, starttime, pSql->sqlstr);
}
void tscSetStreamDestTable(SSqlStream* pStream, const char* dstTable) {
pStream->dstTable = dstTable;
}
// fetchFp call back
void fetchFpStreamLastRow(void* param ,TAOS_RES* res, int num) {
SSqlStream* pStream = (SSqlStream*)param;
SSqlObj* pSql = res;
// get row data set to ltime
tscSetSqlOwner(pSql);
TAOS_ROW row = doSetResultRowData(pSql);
if( row && row[0] ) {
pStream->ltime = *((int64_t*)row[0]);
const char* dstTable = pStream->dstTable? pStream->dstTable: "";
tscDebug(" CQ stream table=%s last row time=%"PRId64" .", dstTable, pStream->ltime);
}
tscClearSqlOwner(pSql);
// no condition call
tscCreateStream(param, pStream->pSql, TSDB_CODE_SUCCESS);
taos_free_result(res);
}
// fp callback
void fpStreamLastRow(void* param ,TAOS_RES* res, int code) {
// check result successful
if (code != TSDB_CODE_SUCCESS) {
tscCreateStream(param, res, TSDB_CODE_SUCCESS);
taos_free_result(res);
return ;
}
// asynchronous fetch last row data
taos_fetch_rows_a(res, fetchFpStreamLastRow, param);
}
void cbParseSql(void* param, TAOS_RES* res, int code) {
// check result successful
SSqlStream* pStream = (SSqlStream*)param;
SSqlObj* pSql = pStream->pSql;
SSqlCmd* pCmd = &pSql->cmd;
if (code != TSDB_CODE_SUCCESS) {
pSql->res.code = code;
tscDebug("0x%"PRIx64" open stream parse sql failed, sql:%s, reason:%s, code:%s", pSql->self, pSql->sqlstr, pCmd->payload, tstrerror(code));
pStream->fp(pStream->param, NULL, NULL);
return;
}
// check dstTable valid
if(pStream->dstTable == NULL || strlen(pStream->dstTable) == 0) {
tscDebug(" cbParseSql dstTable is empty.");
tscCreateStream(param, res, code);
return ;
}
// query stream last row time async
char sql[128] = "";
sprintf(sql, "select last_row(*) from %s;", pStream->dstTable);
taos_query_a(pSql->pTscObj, sql, fpStreamLastRow, param);
}
TAOS_STREAM *taos_open_stream_withname(TAOS *taos, const char* dstTable, const char *sqlstr, void (*fp)(void *, TAOS_RES *, TAOS_ROW),
int64_t stime, void *param, void (*callback)(void *), void* cqhandle) {
STscObj *pObj = (STscObj *)taos;
if (pObj == NULL || pObj->signature != pObj) return NULL;
if(fp == NULL){
tscError(" taos_open_stream api fp param must not NULL.");
return NULL;
}
SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj));
if (pSql == NULL) {
return NULL;
}
pSql->signature = pSql;
pSql->pTscObj = pObj;
SSqlCmd *pCmd = &pSql->cmd;
SSqlRes *pRes = &pSql->res;
SSqlStream *pStream = (SSqlStream *)calloc(1, sizeof(SSqlStream));
if (pStream == NULL) {
tscError("0x%"PRIx64" open stream failed, sql:%s, reason:%s, code:0x%08x", pSql->self, sqlstr, pCmd->payload, pRes->code);
tscFreeSqlObj(pSql);
return NULL;
}
pStream->ltime = INT64_MIN;
pStream->stime = stime;
pStream->fp = fp;
pStream->callback = callback;
pStream->param = param;
pStream->pSql = pSql;
pStream->cqhandle = cqhandle;
tscSetStreamDestTable(pStream, dstTable);
pSql->pStream = pStream;
pSql->param = pStream;
pSql->maxRetry = TSDB_MAX_REPLICA;
pSql->sqlstr = calloc(1, strlen(sqlstr) + 1);
if (pSql->sqlstr == NULL) {
tscError("0x%"PRIx64" failed to malloc sql string buffer", pSql->self);
tscFreeSqlObj(pSql);
free(pStream);
return NULL;
}
strtolower(pSql->sqlstr, sqlstr);
pSql->fp = tscCreateStream;
pSql->fetchFp = tscCreateStream;
pSql->cmd.resColumnId = TSDB_RES_COL_ID;
tsem_init(&pSql->rspSem, 0, 0);
registerSqlObj(pSql);
tscDebugL("0x%"PRIx64" SQL: %s", pSql->self, pSql->sqlstr);
pSql->fp = cbParseSql;
pSql->fetchFp = cbParseSql;
registerSqlObj(pSql);
int32_t code = tsParseSql(pSql, true);
if (code == TSDB_CODE_SUCCESS) {
cbParseSql(pStream, pSql, code);
} else if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
tscDebug("0x%"PRIx64" CQ taso_open_stream IN Process", pSql->self);
} else {
tscError("0x%"PRIx64" open stream failed, sql:%s, code:%s", pSql->self, sqlstr, tstrerror(code));
taosReleaseRef(tscObjRef, pSql->self);
free(pStream);
return NULL;
}
return pStream;
}
TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *, TAOS_RES *, TAOS_ROW),
int64_t stime, void *param, void (*callback)(void *)) {
return taos_open_stream_withname(taos, "", sqlstr, fp, stime, param, callback, NULL);
}
void taos_close_stream(TAOS_STREAM *handle) {
SSqlStream *pStream = (SSqlStream *)handle;
SSqlObj *pSql = (SSqlObj *)atomic_exchange_ptr(&pStream->pSql, 0);
if (pSql == NULL) {
return;
}
/*
* stream may be closed twice, 1. drop dst table, 2. kill stream
* Here, we need a check before release memory
*/
if (pSql->signature == pSql) {
tscRemoveFromStreamList(pStream, pSql);
taosTmrStopA(&(pStream->pTimer));
tscDebug("0x%"PRIx64" stream:%p is closed", pSql->self, pStream);
// notify CQ to release the pStream object
pStream->fp(pStream->param, NULL, NULL);
pStream->pSql = NULL;
taos_free_result(pSql);
tfree(pStream);
}
}

631
2.0/src/client/src/tscSub.c
View File

@ -1,631 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "../../../include/client/taos.h"
#include "os.h"
#include "tcache.h"
#include "trpc.h"
#include "tscLog.h"
#include "tscProfile.h"
#include "tscUtil.h"
#include "tsclient.h"
#include "tsocket.h"
#include "ttimer.h"
#include "tutil.h"
typedef struct SSubscriptionProgress {
int64_t uid;
TSKEY key;
} SSubscriptionProgress;
typedef struct SSub {
void * signature;
char topic[32];
tsem_t sem;
int64_t lastSyncTime;
int64_t lastConsumeTime;
TAOS * taos;
void * pTimer;
SSqlObj * pSql;
int interval;
TAOS_SUBSCRIBE_CALLBACK fp;
void * param;
SArray* progress;
} SSub;
static int tscCompareSubscriptionProgress(const void* a, const void* b) {
const SSubscriptionProgress* x = (const SSubscriptionProgress*)a;
const SSubscriptionProgress* y = (const SSubscriptionProgress*)b;
if (x->uid > y->uid) return 1;
if (x->uid < y->uid) return -1;
return 0;
}
TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid, TSKEY dflt) {
if (sub == NULL) {
return dflt;
}
SSub* pSub = (SSub*)sub;
SSubscriptionProgress target = {.uid = uid, .key = 0};
SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress, TD_EQ);
if (p == NULL) {
return dflt;
}
return p->key;
}
void tscUpdateSubscriptionProgress(void* sub, int64_t uid, TSKEY ts) {
if( sub == NULL) {
return;
}
SSub* pSub = (SSub*)sub;
SSubscriptionProgress target = {.uid = uid, .key = ts};
SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress, TD_EQ);
if (p != NULL) {
p->key = ts;
tscDebug("subscribe:%s, uid:%"PRIu64" update sub start ts:%"PRId64, pSub->topic, p->uid, p->key);
}
}
static void asyncCallback(void *param, TAOS_RES *tres, int code) {
assert(param != NULL);
SSub *pSub = ((SSub *)param);
pSub->pSql->res.code = code;
tsem_post(&pSub->sem);
}
static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char* sql) {
int code = TSDB_CODE_SUCCESS, line = __LINE__;
SSqlObj* pSql = NULL;
SSub* pSub = calloc(1, sizeof(SSub));
if (pSub == NULL) {
line = __LINE__;
code = TSDB_CODE_TSC_OUT_OF_MEMORY;
goto fail;
}
pSub->signature = pSub;
if (tsem_init(&pSub->sem, 0, 0) == -1) {
line = __LINE__;
code = TAOS_SYSTEM_ERROR(errno);
goto fail;
}
tstrncpy(pSub->topic, topic, sizeof(pSub->topic));
pSub->progress = taosArrayInit(32, sizeof(SSubscriptionProgress));
if (pSub->progress == NULL) {
line = __LINE__;
code = TSDB_CODE_TSC_OUT_OF_MEMORY;
goto fail;
}
pSql = calloc(1, sizeof(SSqlObj));
if (pSql == NULL) {
line = __LINE__;
code = TSDB_CODE_TSC_OUT_OF_MEMORY;
goto fail;
}
pSql->signature = pSql;
pSql->pTscObj = pObj;
pSql->pSubscription = pSub;
pSub->pSql = pSql;
SSqlCmd* pCmd = &pSql->cmd;
SSqlRes* pRes = &pSql->res;
if (tsem_init(&pSql->rspSem, 0, 0) == -1) {
line = __LINE__;
code = TAOS_SYSTEM_ERROR(errno);
goto fail;
}
pSql->param = pSub;
pSql->maxRetry = TSDB_MAX_REPLICA;
pSql->fp = asyncCallback;
pSql->fetchFp = asyncCallback;
pSql->sqlstr = strdup(sql);
if (pSql->sqlstr == NULL) {
line = __LINE__;
code = TSDB_CODE_TSC_OUT_OF_MEMORY;
goto fail;
}
strtolower(pSql->sqlstr, pSql->sqlstr);
pRes->qId = 0;
pRes->numOfRows = 1;
pCmd->resColumnId = TSDB_RES_COL_ID;
code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE);
if (code != TSDB_CODE_SUCCESS) {
line = __LINE__;
goto fail;
}
registerSqlObj(pSql);
code = tsParseSql(pSql, true);
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
tsem_wait(&pSub->sem);
code = pSql->res.code;
}
if (code != TSDB_CODE_SUCCESS) {
line = __LINE__;
goto fail;
}
if (pSql->cmd.command != TSDB_SQL_SELECT && pSql->cmd.command != TSDB_SQL_RETRIEVE_EMPTY_RESULT) {
line = __LINE__;
code = TSDB_CODE_TSC_INVALID_OPERATION;
goto fail;
}
return pSub;
fail:
tscError("tscCreateSubscription failed at line %d, reason: %s", line, tstrerror(code));
if (pSql != NULL) {
if (pSql->self != 0) {
taosReleaseRef(tscObjRef, pSql->self);
} else {
tscFreeSqlObj(pSql);
}
pSql = NULL;
}
if (pSub != NULL) {
taosArrayDestroy(pSub->progress);
tsem_destroy(&pSub->sem);
free(pSub);
pSub = NULL;
}
terrno = code;
return NULL;
}
static void tscProcessSubscriptionTimer(void *handle, void *tmrId) {
SSub *pSub = (SSub *)handle;
if (pSub == NULL || pSub->pTimer != tmrId) return;
TAOS_RES* res = taos_consume(pSub);
if (res != NULL) {
pSub->fp(pSub, res, pSub->param, 0);
}
taosTmrReset(tscProcessSubscriptionTimer, pSub->interval, pSub, tscTmr, &pSub->pTimer);
}
//TODO refactor: extract table list name not simply from the sql
static SArray* getTableList( SSqlObj* pSql ) {
const char* p = strstr( pSql->sqlstr, " from " );
assert(p != NULL); // we are sure this is a 'select' statement
char* sql = alloca(strlen(p) + 32);
sprintf(sql, "select tbid(tbname)%s", p);
SSqlObj* pNew = taos_query(pSql->pTscObj, sql);
if (pNew == NULL) {
tscError("0x%"PRIx64" failed to retrieve table id: cannot create new sql object.", pSql->self);
return NULL;
} else if (taos_errno(pNew) != TSDB_CODE_SUCCESS) {
tscError("0x%"PRIx64" failed to retrieve table id,error: %s", pSql->self, tstrerror(taos_errno(pNew)));
return NULL;
}
TAOS_ROW row;
SArray* result = taosArrayInit( 128, sizeof(STidTags) );
while ((row = taos_fetch_row(pNew))) {
STidTags tags;
memcpy(&tags, row[0], sizeof(tags));
taosArrayPush(result, &tags);
}
taos_free_result(pNew);
return result;
}
static int32_t compareTidTag(const void* p1, const void* p2) {
const STidTags* t1 = (const STidTags*)p1;
const STidTags* t2 = (const STidTags*)p2;
if (t1->vgId != t2->vgId) {
return (t1->vgId > t2->vgId) ? 1 : -1;
}
if (t1->tid != t2->tid) {
return (t1->tid > t2->tid) ? 1 : -1;
}
return 0;
}
static int tscUpdateSubscription(STscObj* pObj, SSub* pSub) {
SSqlObj* pSql = pSub->pSql;
SSqlCmd* pCmd = &pSql->cmd;
TSDB_QUERY_CLEAR_TYPE(tscGetQueryInfo(pCmd)->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY);
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0);
if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) {
STableMeta * pTableMeta = pTableMetaInfo->pTableMeta;
SSubscriptionProgress target = {.uid = pTableMeta->id.uid, .key = 0};
SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress, TD_EQ);
if (p == NULL) {
taosArrayClear(pSub->progress);
taosArrayPush(pSub->progress, &target);
}
pSub->lastSyncTime = taosGetTimestampMs();
return 1;
}
SArray* tables = getTableList(pSql);
if (tables == NULL) {
pSub->lastSyncTime = 0; //force to get table list next time
return 0;
}
size_t numOfTables = taosArrayGetSize(tables);
SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd);
SArray* progress = taosArrayInit(numOfTables, sizeof(SSubscriptionProgress));
for( size_t i = 0; i < numOfTables; i++ ) {
STidTags* tt = taosArrayGet( tables, i );
SSubscriptionProgress p = { .uid = tt->uid };
p.key = tscGetSubscriptionProgress(pSub, tt->uid, pQueryInfo->window.skey);
taosArrayPush(progress, &p);
}
taosArraySort(progress, tscCompareSubscriptionProgress);
taosArrayDestroy(pSub->progress);
pSub->progress = progress;
if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
taosArraySort( tables, compareTidTag );
tscFreeVgroupTableInfo(pTableMetaInfo->pVgroupTables);
tscBuildVgroupTableInfo(pSql, pTableMetaInfo, tables);
}
taosArrayDestroy(tables);
if (pTableMetaInfo->pVgroupTables && taosArrayGetSize(pTableMetaInfo->pVgroupTables) > 0) {
TSDB_QUERY_SET_TYPE(tscGetQueryInfo(pCmd)->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY);
}
pSub->lastSyncTime = taosGetTimestampMs();
return 1;
}
static int tscLoadSubscriptionProgress(SSub* pSub) {
char buf[TSDB_MAX_SQL_LEN];
sprintf(buf, "%s/subscribe/%s", tsDataDir, pSub->topic);
FILE* fp = fopen(buf, "rb");
if (fp == NULL) {
tscDebug("subscription progress file does not exist: %s", pSub->topic);
return 1;
}
if (fgets(buf, sizeof(buf), fp) == NULL) {
tscDebug("invalid subscription progress file: %s", pSub->topic);
fclose(fp);
return 0;
}
for (int i = 0; i < sizeof(buf); i++) {
if (buf[i] == 0)
break;
if (buf[i] == '\r' || buf[i] == '\n') {
buf[i] = 0;
break;
}
}
if (strcmp(buf, pSub->pSql->sqlstr) != 0) {
tscDebug("subscription sql statement mismatch: %s", pSub->topic);
fclose(fp);
return 0;
}
SArray* progress = pSub->progress;
taosArrayClear(progress);
while( 1 ) {
if (fgets(buf, sizeof(buf), fp) == NULL) {
fclose(fp);
return 0;
}
SSubscriptionProgress p;
sscanf(buf, "%" SCNd64 ":%" SCNd64, &p.uid, &p.key);
taosArrayPush(progress, &p);
}
fclose(fp);
taosArraySort(progress, tscCompareSubscriptionProgress);
tscDebug("subscription progress loaded, %" PRIzu " tables: %s", taosArrayGetSize(progress), pSub->topic);
return 1;
}
void tscSaveSubscriptionProgress(void* sub) {
SSub* pSub = (SSub*)sub;
char path[256];
sprintf(path, "%s/subscribe", tsDataDir);
if (taosMkDir(path) != 0) {
tscError("failed to create subscribe dir: %s", path);
}
sprintf(path, "%s/subscribe/%s", tsDataDir, pSub->topic);
FILE* fp = fopen(path, "wb+");
if (fp == NULL) {
tscError("failed to create progress file for subscription: %s", pSub->topic);
return;
}
fputs(pSub->pSql->sqlstr, fp);
fprintf(fp, "\n");
for(size_t i = 0; i < taosArrayGetSize(pSub->progress); i++) {
SSubscriptionProgress* p = taosArrayGet(pSub->progress, i);
fprintf(fp, "%" PRId64 ":%" PRId64 "\n", p->uid, p->key);
}
fclose(fp);
}
TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char* topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, void *param, int interval) {
STscObj* pObj = (STscObj*)taos;
if (pObj == NULL || pObj->signature != pObj) {
terrno = TSDB_CODE_TSC_DISCONNECTED;
tscError("connection disconnected");
return NULL;
}
SSub* pSub = tscCreateSubscription(pObj, topic, sql);
if (pSub == NULL) {
return NULL;
}
pSub->taos = taos;
if (restart) {
tscDebug("restart subscription: %s", topic);
} else {
tscLoadSubscriptionProgress(pSub);
}
if (pSub->pSql->cmd.command == TSDB_SQL_SELECT) {
if (!tscUpdateSubscription(pObj, pSub)) {
taos_unsubscribe(pSub, 1);
return NULL;
}
}
pSub->interval = interval;
if (fp != NULL) {
tscDebug("asynchronize subscription, create new timer: %s", topic);
pSub->fp = fp;
pSub->param = param;
taosTmrReset(tscProcessSubscriptionTimer, interval, pSub, tscTmr, &pSub->pTimer);
}
return pSub;
}
SSqlObj* recreateSqlObj(SSub* pSub) {
SSqlObj* pSql = calloc(1, sizeof(SSqlObj));
if (pSql == NULL) {
return NULL;
}
pSql->signature = pSql;
pSql->pTscObj = pSub->taos;
SSqlCmd* pCmd = &pSql->cmd;
SSqlRes* pRes = &pSql->res;
if (tsem_init(&pSql->rspSem, 0, 0) == -1) {
tscFreeSqlObj(pSql);
return NULL;
}
pSql->param = pSub;
pSql->maxRetry = TSDB_MAX_REPLICA;
pSql->fp = asyncCallback;
pSql->fetchFp = asyncCallback;
pSql->sqlstr = strdup(pSub->pSql->sqlstr);
if (pSql->sqlstr == NULL) {
tscFreeSqlObj(pSql);
return NULL;
}
pRes->qId = 0;
pRes->numOfRows = 1;
int code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE);
if (code != TSDB_CODE_SUCCESS) {
tscFreeSqlObj(pSql);
return NULL;
}
registerSqlObj(pSql);
code = tsParseSql(pSql, true);
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
tsem_wait(&pSub->sem);
code = pSql->res.code;
}
if (code != TSDB_CODE_SUCCESS) {
taosReleaseRef(tscObjRef, pSql->self);
return NULL;
}
if (pSql->cmd.command != TSDB_SQL_SELECT) {
taosReleaseRef(tscObjRef, pSql->self);
return NULL;
}
return pSql;
}
TAOS_RES *taos_consume(TAOS_SUB *tsub) {
SSub *pSub = (SSub *)tsub;
if (pSub == NULL) return NULL;
if (pSub->pTimer == NULL) {
int64_t duration = taosGetTimestampMs() - pSub->lastConsumeTime;
if (duration < (int64_t)(pSub->interval)) {
tscDebug("subscription consume too frequently, blocking...");
taosMsleep(pSub->interval - (int32_t)duration);
}
}
if (pSub->pSql->cmd.command == TSDB_SQL_RETRIEVE_EMPTY_RESULT) { //may reach here when retireve stable vgroup failed
SSqlObj* pSql = recreateSqlObj(pSub);
if (pSql == NULL) {
return NULL;
}
if (pSub->pSql->self != 0) {
taosReleaseRef(tscObjRef, pSub->pSql->self);
} else {
tscFreeSqlObj(pSub->pSql);
}
pSub->pSql = pSql;
pSql->pSubscription = pSub;
pSub->lastSyncTime = 0;
// no table list now, force to update it
tscDebug("begin table synchronization");
if (!tscUpdateSubscription(pSub->taos, pSub)) return NULL;
tscDebug("table synchronization completed");
}
tscSaveSubscriptionProgress(pSub);
SSqlObj *pSql = pSub->pSql;
SSqlRes *pRes = &pSql->res;
SSqlCmd *pCmd = &pSql->cmd;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0);
SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd);
if (taosArrayGetSize(pSub->progress) > 0) { // fix crash in single table subscription
size_t size = taosArrayGetSize(pSub->progress);
TSKEY s = INT64_MAX;
for(int32_t i = 0; i < size; ++i) {
TSKEY k = ((SSubscriptionProgress*)taosArrayGet(pSub->progress, i))->key;
if (s > k) {
s = k;
}
}
pQueryInfo->window.skey = s;
tscDebug("subscribe:%s set next round subscribe skey:%"PRId64, pSub->topic, pQueryInfo->window.skey);
}
size_t size = taosArrayGetSize(pSub->progress) * sizeof(STableIdInfo);
size += sizeof(SQueryTableMsg) + 4096;
int code = tscAllocPayload(&pSql->cmd, (int)size);
if (code != TSDB_CODE_SUCCESS) {
tscError("failed to alloc payload");
return NULL;
}
for (int retry = 0; retry < 3; retry++) {
tscRemoveFromSqlList(pSql);
if (taosGetTimestampMs() - pSub->lastSyncTime > 10 * 60 * 1000) {
tscDebug("begin table synchronization");
if (!tscUpdateSubscription(pSub->taos, pSub)) return NULL;
tscDebug("table synchronization completed");
}
uint32_t type = pQueryInfo->type;
tscFreeSqlResult(pSql);
pRes->numOfRows = 1;
pRes->qId = 0;
pSql->cmd.command = TSDB_SQL_SELECT;
pQueryInfo->type = type;
pTableMetaInfo->vgroupIndex = 0;
pSql->fp = asyncCallback;
pSql->fetchFp = asyncCallback;
pSql->param = pSub;
pSql->cmd.active = pQueryInfo;
executeQuery(pSql, pQueryInfo);
tsem_wait(&pSub->sem);
if (pRes->code != TSDB_CODE_SUCCESS) {
continue;
}
// meter was removed, make sync time zero, so that next retry will
// do synchronization first
pSub->lastSyncTime = 0;
break;
}
if (pRes->code != TSDB_CODE_SUCCESS) {
tscError("failed to query data: %s", tstrerror(pRes->code));
tscRemoveFromSqlList(pSql);
return NULL;
}
pSub->lastConsumeTime = taosGetTimestampMs();
return pSql;
}
void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) {
SSub *pSub = (SSub *)tsub;
if (pSub == NULL || pSub->signature != pSub) return;
if (pSub->pTimer != NULL) {
taosTmrStop(pSub->pTimer);
}
if (keepProgress) {
if (pSub->progress != NULL) {
tscSaveSubscriptionProgress(pSub);
}
} else {
char path[256];
sprintf(path, "%s/subscribe/%s", tsDataDir, pSub->topic);
if (remove(path) != 0) {
tscError("failed to remove progress file, topic = %s, error = %s", pSub->topic, strerror(errno));
}
}
if (pSub->pSql != NULL) {
if (pSub->pSql->self != 0) {
taosReleaseRef(tscObjRef, pSub->pSql->self);
} else {
tscFreeSqlObj(pSub->pSql);
}
}
taosArrayDestroy(pSub->progress);
tsem_destroy(&pSub->sem);
memset(pSub, 0, sizeof(*pSub));
free(pSub);
}

3737
2.0/src/client/src/tscSubquery.c
View File

File diff suppressed because it is too large Load Diff

439
2.0/src/client/src/tscSystem.c
View File

@ -1,439 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "tmsg.h"
#include "tconfig.h"
#include "tglobal.h"
#include "tnote.h"
#include "tref.h"
#include "trpc.h"
#include "tscLog.h"
#include "tsched.h"
#include "tsclient.h"
#include "tscript.h"
#include "ttimer.h"
#include "ttimezone.h"
// global, not configurable
#define TSC_VAR_NOT_RELEASE 1
#define TSC_VAR_RELEASED 0
int32_t sentinel = TSC_VAR_NOT_RELEASE;
SHashObj *tscVgroupMap; // hash map to keep the vgroup info from mnode
SHashObj *tscTableMetaMap; // table meta info buffer
SCacheObj *tscVgroupListBuf; // super table vgroup list information, only survives 5 seconds for each super table vgroup list
int32_t tscObjRef = -1;
void *tscTmr;
void *tscQhandle;
int32_t tscRefId = -1;
int32_t tscNumOfObj = 0; // number of sqlObj in current process.
static void *tscCheckDiskUsageTmr;
void *tscRpcCache; // cache to keep rpc obj
int32_t tscNumOfThreads = 1; // num of rpc threads
char tscLogFileName[12] = "taoslog";
int tscLogFileNum = 10;
static pthread_mutex_t rpcObjMutex; // mutex to protect open the rpc obj concurrently
static pthread_once_t tscinit = PTHREAD_ONCE_INIT;
// pthread_once can not return result code, so result code is set to a global variable.
static volatile int tscInitRes = 0;
void tscCheckDiskUsage(void *UNUSED_PARAM(para), void *UNUSED_PARAM(param)) {
taosGetDisk();
taosTmrReset(tscCheckDiskUsage, 20 * 1000, NULL, tscTmr, &tscCheckDiskUsageTmr);
}
void tscFreeRpcObj(void *param) {
assert(param);
SRpcObj *pRpcObj = (SRpcObj *)(param);
tscDebug("free rpcObj:%p and free pDnodeConn: %p", pRpcObj, pRpcObj->pDnodeConn);
rpcClose(pRpcObj->pDnodeConn);
}
void tscReleaseRpc(void *param) {
if (param == NULL) {
return;
}
taosCacheRelease(tscRpcCache, (void *)&param, false);
}
int32_t tscAcquireRpc(const char *key, const char *user, const char *secretEncrypt, void **ppRpcObj) {
pthread_mutex_lock(&rpcObjMutex);
SRpcObj *pRpcObj = (SRpcObj *)taosCacheAcquireByKey(tscRpcCache, key, strlen(key));
if (pRpcObj != NULL) {
*ppRpcObj = pRpcObj;
pthread_mutex_unlock(&rpcObjMutex);
return 0;
}
SRpcInit rpcInit;
memset(&rpcInit, 0, sizeof(rpcInit));
rpcInit.localPort = 0;
rpcInit.label = "TSC";
rpcInit.numOfThreads = tscNumOfThreads;
rpcInit.cfp = tscProcessMsgFromServer;
rpcInit.sessions = tsMaxConnections;
rpcInit.connType = TAOS_CONN_CLIENT;
rpcInit.user = (char *)user;
rpcInit.idleTime = tsShellActivityTimer * 1000;
rpcInit.ckey = "key";
rpcInit.spi = 1;
rpcInit.secret = (char *)secretEncrypt;
SRpcObj rpcObj;
memset(&rpcObj, 0, sizeof(rpcObj));
strncpy(rpcObj.key, key, strlen(key));
rpcObj.pDnodeConn = rpcOpen(&rpcInit);
if (rpcObj.pDnodeConn == NULL) {
pthread_mutex_unlock(&rpcObjMutex);
tscError("failed to init connection to TDengine");
return -1;
}
pRpcObj = taosCachePut(tscRpcCache, rpcObj.key, strlen(rpcObj.key), &rpcObj, sizeof(rpcObj), 1000*5);
if (pRpcObj == NULL) {
rpcClose(rpcObj.pDnodeConn);
pthread_mutex_unlock(&rpcObjMutex);
return -1;
}
*ppRpcObj = pRpcObj;
pthread_mutex_unlock(&rpcObjMutex);
return 0;
}
void taos_init_imp(void) {
char temp[128] = {0};
// In the APIs of other program language, taos_cleanup is not available yet.
// So, to make sure taos_cleanup will be invoked to clean up the allocated resource to suppress the valgrind warning.
atexit(taos_cleanup);
errno = TSDB_CODE_SUCCESS;
srand(taosGetTimestampSec());
deltaToUtcInitOnce();
if (tscEmbedded == 0) {
// Read global configuration.
taosInitGlobalCfg();
taosReadGlobalLogCfg();
// For log directory
if (mkdir(tsLogDir, 0755) != 0 && errno != EEXIST) {
printf("failed to create log dir:%s\n", tsLogDir);
}
sprintf(temp, "%s/%s", tsLogDir, tscLogFileName);
if (taosInitLog(temp, tsNumOfLogLines, tscLogFileNum) < 0) {
printf("failed to open log file in directory:%s\n", tsLogDir);
}
taosReadGlobalCfg();
if (taosCheckGlobalCfg() != 0) {
tscInitRes = -1;
return;
}
taosInitNotes();
rpcInit();
scriptEnvPoolInit();
tscDebug("starting to initialize TAOS client ...");
tscDebug("Local End Point is:%s", tsLocalEp);
}
taosSetCoreDump();
tscInitMsgsFp();
double factor = (tscEmbedded == 0)? 2.0:4.0;
tscNumOfThreads = (int)(tsNumOfCores * tsNumOfThreadsPerCore / factor);
if (tscNumOfThreads < 2) {
tscNumOfThreads = 2;
}
int32_t queueSize = tsMaxConnections*2;
tscQhandle = taosInitScheduler(queueSize, tscNumOfThreads, "tsc");
if (NULL == tscQhandle) {
tscError("failed to init task queue");
tscInitRes = -1;
return;
}
tscDebug("client task queue is initialized, numOfWorkers: %d", tscNumOfThreads);
tscTmr = taosTmrInit(tsMaxConnections * 2, 200, 60000, "TSC");
if(0 == tscEmbedded){
taosTmrReset(tscCheckDiskUsage, 20 * 1000, NULL, tscTmr, &tscCheckDiskUsageTmr);
}
if (tscTableMetaMap == NULL) {
tscObjRef = taosOpenRef(40960, tscFreeRegisteredSqlObj);
tscVgroupMap = taosHashInit(256, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
tscTableMetaMap = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
tscVgroupListBuf = taosCacheInit(TSDB_DATA_TYPE_BINARY, 5, false, NULL, "stable-vgroup-list");
tscDebug("TableMeta:%p, vgroup:%p is initialized", tscTableMetaMap, tscVgroupMap);
}
int refreshTime = 5;
tscRpcCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, true, tscFreeRpcObj, "rpcObj");
pthread_mutex_init(&rpcObjMutex, NULL);
tscRefId = taosOpenRef(200, tscCloseTscObj);
tscDebug("client is initialized successfully");
}
int taos_init() {
pthread_once(&tscinit, taos_init_imp);
return tscInitRes;
}
// this function may be called by user or system, or by both simultaneously.
void taos_cleanup(void) {
tscDebug("start to cleanup client environment");
if (atomic_val_compare_exchange_32(&sentinel, TSC_VAR_NOT_RELEASE, TSC_VAR_RELEASED) != TSC_VAR_NOT_RELEASE) {
return;
}
if (tscEmbedded == 0) {
scriptEnvPoolCleanup();
}
taosHashCleanup(tscTableMetaMap);
tscTableMetaMap = NULL;
taosHashCleanup(tscVgroupMap);
tscVgroupMap = NULL;
int32_t id = tscObjRef;
tscObjRef = -1;
taosCloseRef(id);
void* p = tscQhandle;
tscQhandle = NULL;
taosCleanUpScheduler(p);
id = tscRefId;
tscRefId = -1;
taosCloseRef(id);
taosCleanupKeywordsTable();
p = tscRpcCache;
tscRpcCache = NULL;
if (p != NULL) {
taosCacheCleanup(p);
pthread_mutex_destroy(&rpcObjMutex);
}
taosCacheCleanup(tscVgroupListBuf);
tscVgroupListBuf = NULL;
if (tscEmbedded == 0) {
rpcCleanup();
taosCloseLog();
};
p = tscTmr;
tscTmr = NULL;
taosTmrCleanUp(p);
}
static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
SGlobalCfg *cfg = NULL;
switch (option) {
case TSDB_OPTION_CONFIGDIR:
cfg = taosGetConfigOption("configDir");
assert(cfg != NULL);
if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) {
tstrncpy(configDir, pStr, TSDB_FILENAME_LEN);
cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
tscInfo("set config file directory:%s", pStr);
} else {
tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, pStr,
tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr);
}
break;
case TSDB_OPTION_SHELL_ACTIVITY_TIMER:
cfg = taosGetConfigOption("shellActivityTimer");
assert(cfg != NULL);
if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) {
tsShellActivityTimer = atoi(pStr);
if (tsShellActivityTimer < 1) tsShellActivityTimer = 1;
if (tsShellActivityTimer > 3600) tsShellActivityTimer = 3600;
cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
tscInfo("set shellActivityTimer:%d", tsShellActivityTimer);
} else {
tscWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, pStr,
tsCfgStatusStr[cfg->cfgStatus], *(int32_t *)cfg->ptr);
}
break;
case TSDB_OPTION_LOCALE: { // set locale
cfg = taosGetConfigOption("locale");
assert(cfg != NULL);
size_t len = strlen(pStr);
if (len == 0 || len > TSDB_LOCALE_LEN) {
tscInfo("Invalid locale:%s, use default", pStr);
return -1;
}
if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) {
char sep = '.';
if (strlen(tsLocale) == 0) { // locale does not set yet
char* defaultLocale = setlocale(LC_CTYPE, "");
// The locale of the current OS does not be set correctly, so the default locale cannot be acquired.
// The launch of current system will abort soon.
if (defaultLocale == NULL) {
tscError("failed to get default locale, please set the correct locale in current OS");
return -1;
}
tstrncpy(tsLocale, defaultLocale, TSDB_LOCALE_LEN);
}
// set the user specified locale
char *locale = setlocale(LC_CTYPE, pStr);
if (locale != NULL) { // failed to set the user specified locale
tscInfo("locale set, prev locale:%s, new locale:%s", tsLocale, locale);
cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
} else { // set the user specified locale failed, use default LC_CTYPE as current locale
locale = setlocale(LC_CTYPE, tsLocale);
tscInfo("failed to set locale:%s, current locale:%s", pStr, tsLocale);
}
tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN);
char *charset = strrchr(tsLocale, sep);
if (charset != NULL) {
charset += 1;
charset = taosCharsetReplace(charset);
if (taosValidateEncodec(charset)) {
if (strlen(tsCharset) == 0) {
tscInfo("charset set:%s", charset);
} else {
tscInfo("charset changed from %s to %s", tsCharset, charset);
}
tstrncpy(tsCharset, charset, TSDB_LOCALE_LEN);
cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
} else {
tscInfo("charset:%s is not valid in locale, charset remains:%s", charset, tsCharset);
}
free(charset);
} else { // it may be windows system
tscInfo("charset remains:%s", tsCharset);
}
} else {
tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, pStr,
tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr);
}
break;
}
case TSDB_OPTION_CHARSET: {
/* set charset will override the value of charset, assigned during system locale changed */
cfg = taosGetConfigOption("charset");
assert(cfg != NULL);
size_t len = strlen(pStr);
if (len == 0 || len > TSDB_LOCALE_LEN) {
tscInfo("failed to set charset:%s", pStr);
return -1;
}
if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) {
if (taosValidateEncodec(pStr)) {
if (strlen(tsCharset) == 0) {
tscInfo("charset is set:%s", pStr);
} else {
tscInfo("charset changed from %s to %s", tsCharset, pStr);
}
tstrncpy(tsCharset, pStr, TSDB_LOCALE_LEN);
cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
} else {
tscInfo("charset:%s not valid", pStr);
}
} else {
tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, pStr,
tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr);
}
break;
}
case TSDB_OPTION_TIMEZONE:
cfg = taosGetConfigOption("timezone");
assert(cfg != NULL);
if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) {
tstrncpy(tsTimezone, pStr, TSDB_TIMEZONE_LEN);
tsSetTimeZone();
cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
tscDebug("timezone set:%s, input:%s by taos_options", tsTimezone, pStr);
} else {
tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, pStr,
tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr);
}
break;
default:
// TODO return the correct error code to client in the format for taos_errstr()
tscError("Invalid option %d", option);
return -1;
}
return 0;
}
int taos_options(TSDB_OPTION option, const void *arg, ...) {
static int32_t lock = 0;
for (int i = 1; atomic_val_compare_exchange_32(&lock, 0, 1) != 0; ++i) {
if (i % 1000 == 0) {
tscInfo("haven't acquire lock after spin %d times.", i);
sched_yield();
}
}
int ret = taos_options_imp(option, (const char*)arg);
atomic_store_32(&lock, 0);
return ret;
}

5132
2.0/src/client/src/tscUtil.c
View File

File diff suppressed because it is too large Load Diff

21
2.0/src/client/tests/CMakeLists.txt
View File

@ -1,21 +0,0 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20)
PROJECT(TDengine)
FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest)
FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib /usr/lib64)
FIND_LIBRARY(LIB_GTEST_SHARED_DIR libgtest.so /usr/lib/ /usr/local/lib /usr/lib64)
IF (HEADER_GTEST_INCLUDE_DIR AND (LIB_GTEST_STATIC_DIR OR LIB_GTEST_SHARED_DIR))
MESSAGE(STATUS "gTest library found, build unit test")
# GoogleTest requires at least C++11
SET(CMAKE_CXX_STANDARD 11)
INCLUDE_DIRECTORIES(/usr/include /usr/local/include)
LINK_DIRECTORIES(/usr/lib /usr/local/lib)
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
ADD_EXECUTABLE(cliTest ${SOURCE_LIST})
TARGET_LINK_LIBRARIES(cliTest taos tutil common gtest pthread)
ENDIF()

267
2.0/src/client/tests/cliTest.cpp
View File

@ -1,267 +0,0 @@
#include <gtest/gtest.h>
#include <iostream>
#include <inttypes.h>
#include "taos.h"
#include "tglobal.h"
namespace {
static int64_t start_ts = 1433955661000;
void stmtInsertTest() {
TAOS* conn = taos_connect("ubuntu", "root", "taosdata", 0, 0);
if (conn == NULL) {
printf("Failed to connect to DB, reason:%s", taos_errstr(conn));
exit(-1);
}
TAOS_RES* res = taos_query(conn, "use test");
taos_free_result(res);
const char* sql = "insert into t1 values(?, ?, ?, ?)";
TAOS_STMT* stmt = taos_stmt_init(conn);
int32_t ret = taos_stmt_prepare(stmt, sql, 0);
ASSERT_EQ(ret, 0);
//ts timestamp, k int, a binary(11), b nchar(4)
struct {
int64_t ts;
int k;
char* a;
char* b;
} v = {0};
TAOS_BIND params[4];
params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP;
params[0].buffer_length = sizeof(v.ts);
params[0].buffer = &v.ts;
params[0].length = &params[0].buffer_length;
params[0].is_null = NULL;
params[1].buffer_type = TSDB_DATA_TYPE_INT;
params[1].buffer_length = sizeof(v.k);
params[1].buffer = &v.k;
params[1].length = &params[1].buffer_length;
params[1].is_null = NULL;
params[2].buffer_type = TSDB_DATA_TYPE_BINARY;
params[2].buffer_length = sizeof(v.a);
params[2].buffer = &v.a;
params[2].is_null = NULL;
params[3].buffer_type = TSDB_DATA_TYPE_NCHAR;
params[3].buffer_length = sizeof(v.b);
params[3].buffer = &v.b;
params[3].is_null = NULL;
v.ts = start_ts + 20;
v.k = 123;
char str[] = "abc";
uintptr_t len = strlen(str);
v.a = str;
params[2].length = &len;
params[2].buffer_length = len;
params[2].buffer = str;
char nstr[] = "999";
uintptr_t len1 = strlen(nstr);
v.b = nstr;
params[3].buffer_length = len1;
params[3].buffer = nstr;
params[3].length = &len1;
taos_stmt_bind_param(stmt, params);
taos_stmt_add_batch(stmt);
if (taos_stmt_execute(stmt) != 0) {
printf("\033[31mfailed to execute insert statement.\033[0m\n");
return;
}
v.ts = start_ts + 30;
v.k = 911;
char str1[] = "92";
len = strlen(str1);
params[2].length = &len;
params[2].buffer_length = len;
params[2].buffer = str1;
char nstr1[] = "1920";
len1 = strlen(nstr1);
params[3].buffer_length = len1;
params[3].buffer = nstr1;
params[3].length = &len1;
taos_stmt_bind_param(stmt, params);
taos_stmt_add_batch(stmt);
ret = taos_stmt_execute(stmt);
if (ret != 0) {
printf("%d\n", ret);
printf("\033[31mfailed to execute insert statement.\033[0m\n");
return;
}
taos_stmt_close(stmt);
taos_close(conn);
}
void validateResultFields() {
TAOS* conn = taos_connect("ubuntu", "root", "taosdata", 0, 0);
if (conn == NULL) {
printf("Failed to connect to DB, reason:%s", taos_errstr(conn));
exit(-1);
}
TAOS_RES* res = taos_query(conn, "create database if not exists test");
ASSERT_EQ(taos_errno(res), 0);
taos_free_result(res);
res = taos_query(conn, "use test");
ASSERT_EQ(taos_errno(res), 0);
taos_free_result(res);
res = taos_query(conn, "create table if not exists t1(ts timestamp, k int, a binary(11), b nchar(4))");
ASSERT_EQ(taos_errno(res), 0);
taos_free_result(res);
char sql[512] = {0};
sprintf(sql, "insert into t1 values(%" PRId64 ", 99, 'abc', 'test')", start_ts);
res = taos_query(conn, sql);
ASSERT_EQ(taos_errno(res), 0);
taos_free_result(res);
res = taos_query(conn, "select count(*), spread(ts)/(1000 * 3600 * 24), first(a), last(b) from t1");
ASSERT_EQ(taos_num_fields(res), 4);
TAOS_FIELD* fields = taos_fetch_fields(res);
ASSERT_EQ(fields[0].bytes, 8);
ASSERT_EQ(fields[0].type, TSDB_DATA_TYPE_BIGINT);
ASSERT_STREQ(fields[0].name, "count(*)");
ASSERT_EQ(fields[1].bytes, 8);
ASSERT_EQ(fields[1].type, TSDB_DATA_TYPE_DOUBLE);
ASSERT_STREQ(fields[1].name, "spread(ts)/(1000 * 3600 * 24)");
ASSERT_EQ(fields[2].bytes, 11);
ASSERT_EQ(fields[2].type, TSDB_DATA_TYPE_BINARY);
ASSERT_STREQ(fields[2].name, "first(a)");
ASSERT_EQ(fields[3].bytes, 4);
ASSERT_EQ(fields[3].type, TSDB_DATA_TYPE_NCHAR);
ASSERT_STREQ(fields[3].name, "last(b)");
taos_free_result(res);
res = taos_query(conn, "select last_row(*) from t1");
ASSERT_EQ(taos_num_fields(res), 4);
fields = taos_fetch_fields(res);
ASSERT_EQ(fields[0].bytes, 8);
ASSERT_EQ(fields[0].type, TSDB_DATA_TYPE_TIMESTAMP);
ASSERT_STREQ(fields[0].name, "last_row(ts)");
ASSERT_EQ(fields[1].bytes, 4);
ASSERT_EQ(fields[1].type, TSDB_DATA_TYPE_INT);
ASSERT_STREQ(fields[1].name, "last_row(k)");
ASSERT_EQ(fields[2].bytes, 11);
ASSERT_EQ(fields[2].type, TSDB_DATA_TYPE_BINARY);
ASSERT_STREQ(fields[2].name, "last_row(a)");
ASSERT_EQ(fields[3].bytes, 4);
ASSERT_EQ(fields[3].type, TSDB_DATA_TYPE_NCHAR);
ASSERT_STREQ(fields[3].name, "last_row(b)");
taos_free_result(res);
res = taos_query(conn, "select first(*), last(*) from t1");
ASSERT_EQ(taos_num_fields(res), 8);
fields = taos_fetch_fields(res);
ASSERT_EQ(fields[0].bytes, 8);
ASSERT_EQ(fields[0].type, TSDB_DATA_TYPE_TIMESTAMP);
ASSERT_STREQ(fields[0].name, "first(ts)");
ASSERT_EQ(fields[1].bytes, 4);
ASSERT_EQ(fields[1].type, TSDB_DATA_TYPE_INT);
ASSERT_STREQ(fields[1].name, "first(k)");
ASSERT_EQ(fields[2].bytes, 11);
ASSERT_EQ(fields[2].type, TSDB_DATA_TYPE_BINARY);
ASSERT_STREQ(fields[2].name, "first(a)");
ASSERT_EQ(fields[3].bytes, 4);
ASSERT_EQ(fields[3].type, TSDB_DATA_TYPE_NCHAR);
ASSERT_STREQ(fields[3].name, "first(b)");
taos_free_result(res);
res = taos_query(conn, "select first(ts, a, k, k, b, b, ts) from t1");
ASSERT_EQ(taos_num_fields(res), 7);
fields = taos_fetch_fields(res);
ASSERT_EQ(fields[0].bytes, 8);
ASSERT_EQ(fields[0].type, TSDB_DATA_TYPE_TIMESTAMP);
ASSERT_STREQ(fields[0].name, "first(ts)");
ASSERT_EQ(fields[1].bytes, 11);
ASSERT_EQ(fields[1].type, TSDB_DATA_TYPE_BINARY);
ASSERT_STREQ(fields[1].name, "first(a)");
ASSERT_EQ(fields[2].bytes, 4);
ASSERT_EQ(fields[2].type, TSDB_DATA_TYPE_INT);
ASSERT_STREQ(fields[2].name, "first(k)");
ASSERT_EQ(fields[3].bytes, 4);
ASSERT_EQ(fields[3].type, TSDB_DATA_TYPE_INT);
ASSERT_STREQ(fields[3].name, "first(k)");
ASSERT_EQ(fields[4].bytes, 4);
ASSERT_EQ(fields[4].type, TSDB_DATA_TYPE_NCHAR);
ASSERT_STREQ(fields[4].name, "first(b)");
ASSERT_EQ(fields[5].bytes, 4);
ASSERT_EQ(fields[5].type, TSDB_DATA_TYPE_NCHAR);
ASSERT_STREQ(fields[5].name, "first(b)");
ASSERT_EQ(fields[6].bytes, 8);
ASSERT_EQ(fields[6].type, TSDB_DATA_TYPE_TIMESTAMP);
ASSERT_STREQ(fields[6].name, "first(ts)");
taos_free_result(res);
// update the configure parameter, the result field name will be changed
tsKeepOriginalColumnName = 1;
res = taos_query(conn, "select first(ts, a, k, k, b, b, ts) from t1");
ASSERT_EQ(taos_num_fields(res), 7);
fields = taos_fetch_fields(res);
ASSERT_EQ(fields[0].bytes, 8);
ASSERT_EQ(fields[0].type, TSDB_DATA_TYPE_TIMESTAMP);
ASSERT_STREQ(fields[0].name, "ts");
ASSERT_EQ(fields[2].bytes, 4);
ASSERT_EQ(fields[2].type, TSDB_DATA_TYPE_INT);
ASSERT_STREQ(fields[2].name, "k");
taos_free_result(res);
taos_close(conn);
}
}
/* test parse time function */
TEST(testCase, result_field_test) {
taos_options(TSDB_OPTION_CONFIGDIR, "~/first/cfg");
taos_init();
validateResultFields();
stmtInsertTest();
}

173
2.0/src/client/tests/timeParseTest.cpp
View File

@ -1,173 +0,0 @@
#include <gtest/gtest.h>
#include <cassert>
#include <iostream>
#include <inttypes.h>
#include "os.h"
#include "taos.h"
#include "ttoken.h"
#include "tutil.h"
int main(int argc, char** argv) {
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
extern void deltaToUtcInitOnce();
/* test parse time function */
TEST(testCase, parse_time) {
taos_options(TSDB_OPTION_TIMEZONE, "GMT-8");
deltaToUtcInitOnce();
char t1[] = "2018-1-1 1:1:1.952798";
char t13[] = "1970-1-1 0:0:0";
int64_t time = 0, time1 = 0;
taosParseTime(t1, &time, strlen(t1), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, 1514739661952);
taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, timezone * MILLISECOND_PER_SECOND);
char t2[] = "2018-1-1T1:1:1.952Z";
taosParseTime(t2, &time, strlen(t2), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, 1514739661952 + 28800000);
char t3[] = "2018-1-1 1:01:01.952";
taosParseTime(t3, &time, strlen(t3), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, 1514739661952);
char t4[] = "2018-1-1 1:01:01.9";
char t5[] = "2018-1-1 1:01:1.900";
char t6[] = "2018-01-01 1:1:1.90";
char t7[] = "2018-01-01 01:01:01.9";
char t8[] = "2018-01-01 01:01:01.9007865";
taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI, 0);
taosParseTime(t5, &time1, strlen(t5), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, time1);
taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI, 0);
taosParseTime(t6, &time1, strlen(t6), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, time1);
taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI, 0);
taosParseTime(t7, &time1, strlen(t7), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, time1);
taosParseTime(t5, &time, strlen(t5), TSDB_TIME_PRECISION_MILLI, 0);
taosParseTime(t8, &time1, strlen(t8), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, time1);
char t9[] = "2017-4-3 1:1:2.980";
char t10[] = "2017-4-3T2:1:2.98+9:00";
taosParseTime(t9, &time, strlen(t9), TSDB_TIME_PRECISION_MILLI, 0);
taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, time1);
char t11[] = "2017-4-3T2:1:2.98+09:00";
taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI, 0);
taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, time1);
char t12[] = "2017-4-3T2:1:2.98+0900";
taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI, 0);
taosParseTime(t12, &time1, strlen(t12), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, time1);
taos_options(TSDB_OPTION_TIMEZONE, "UTC");
deltaToUtcInitOnce();
taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, 0);
taos_options(TSDB_OPTION_TIMEZONE, "Asia/Shanghai");
deltaToUtcInitOnce();
char t14[] = "1970-1-1T0:0:0Z";
taosParseTime(t14, &time, strlen(t14), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, 0);
char t40[] = "1970-1-1 0:0:0.999999999";
taosParseTime(t40, &time, strlen(t40), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, 999 + timezone * MILLISECOND_PER_SECOND);
char t41[] = "1997-1-1 0:0:0.999999999";
taosParseTime(t41, &time, strlen(t41), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, 852048000999);
// int64_t k = timezone;
char t42[] = "1997-1-1T0:0:0.999999999Z";
taosParseTime(t42, &time, strlen(t42), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, 852048000999 - timezone * MILLISECOND_PER_SECOND);
////////////////////////////////////////////////////////////////////
// illegal timestamp format
char t15[] = "2017-12-33 0:0:0";
EXPECT_EQ(taosParseTime(t15, &time, strlen(t15), TSDB_TIME_PRECISION_MILLI, 0), -1);
char t16[] = "2017-12-31 99:0:0";
EXPECT_EQ(taosParseTime(t16, &time, strlen(t16), TSDB_TIME_PRECISION_MILLI, 0), -1);
char t17[] = "2017-12-31T9:0:0";
EXPECT_EQ(taosParseTime(t17, &time, strlen(t17), TSDB_TIME_PRECISION_MILLI, 0), -1);
char t18[] = "2017-12-31T9:0:0.Z";
EXPECT_EQ(taosParseTime(t18, &time, strlen(t18), TSDB_TIME_PRECISION_MILLI, 0), -1);
char t19[] = "2017-12-31 9:0:0.-1";
EXPECT_EQ(taosParseTime(t19, &time, strlen(t19), TSDB_TIME_PRECISION_MILLI, 0), -1);
char t20[] = "2017-12-31 9:0:0.1+12:99";
EXPECT_EQ(taosParseTime(t20, &time, strlen(t20), TSDB_TIME_PRECISION_MILLI, 0), 0);
EXPECT_EQ(time, 1514682000100);
char t21[] = "2017-12-31T9:0:0.1+12:99";
EXPECT_EQ(taosParseTime(t21, &time, strlen(t21), TSDB_TIME_PRECISION_MILLI, 0), -1);
char t22[] = "2017-12-31 9:0:0.1+13:1";
EXPECT_EQ(taosParseTime(t22, &time, strlen(t22), TSDB_TIME_PRECISION_MILLI, 0), 0);
char t23[] = "2017-12-31T9:0:0.1+13:1";
EXPECT_EQ(taosParseTime(t23, &time, strlen(t23), TSDB_TIME_PRECISION_MILLI, 0), 0);
//======================== add some case ============================//
char b1[] = "9999-12-31 23:59:59.999";
taosParseTime(b1, &time, strlen(b1), TSDB_TIME_PRECISION_MILLI,0);
EXPECT_EQ(time, 253402271999999);
char b2[] = "2020-01-01 01:01:01.321";
taosParseTime(b2, &time, strlen(b2), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, 1577811661321);
taos_options(TSDB_OPTION_TIMEZONE, "America/New_York");
deltaToUtcInitOnce();
taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, 18000 * MILLISECOND_PER_SECOND);
taos_options(TSDB_OPTION_TIMEZONE, "Asia/Tokyo");
deltaToUtcInitOnce();
taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, -32400 * MILLISECOND_PER_SECOND);
taos_options(TSDB_OPTION_TIMEZONE, "Asia/Shanghai");
deltaToUtcInitOnce();
taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, -28800 * MILLISECOND_PER_SECOND);
char t[] = "2021-01-08T02:11:40.000+00:00";
taosParseTime(t, &time, strlen(t), TSDB_TIME_PRECISION_MILLI, 0);
printf("%" PRId64 "\n", time);
}

21
2.0/src/query/CMakeLists.txt
View File

@ -1,21 +0,0 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20)
PROJECT(TDengine)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/tsdb/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
INCLUDE_DIRECTORIES(inc)
AUX_SOURCE_DIRECTORY(src SRC)
ADD_LIBRARY(query ${SRC})
SET_SOURCE_FILES_PROPERTIES(src/sql.c PROPERTIES COMPILE_FLAGS -w)
TARGET_LINK_LIBRARIES(query tsdb tutil lua)
IF (TD_LINUX)
TARGET_LINK_LIBRARIES(query m rt lua)
ADD_SUBDIRECTORY(tests)
ENDIF ()
IF (TD_DARWIN)
TARGET_LINK_LIBRARIES(query m lua)
ADD_SUBDIRECTORY(tests)
ENDIF ()

275
2.0/src/query/inc/qAggMain.h
View File

@ -1,275 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_QAGGMAIN_H
#define TDENGINE_QAGGMAIN_H
#ifdef __cplusplus
extern "C" {
#endif
#include "os.h"
#include "tname.h"
#include "taosdef.h"
#include "trpc.h"
#include "tvariant.h"
#include "tsdb.h"
#include "qUdf.h"
#define TSDB_FUNC_INVALID_ID -1
#define TSDB_FUNC_COUNT 0
#define TSDB_FUNC_SUM 1
#define TSDB_FUNC_AVG 2
#define TSDB_FUNC_MIN 3
#define TSDB_FUNC_MAX 4
#define TSDB_FUNC_STDDEV 5
#define TSDB_FUNC_PERCT 6
#define TSDB_FUNC_APERCT 7
#define TSDB_FUNC_FIRST 8
#define TSDB_FUNC_LAST 9
#define TSDB_FUNC_LAST_ROW 10
#define TSDB_FUNC_TOP 11
#define TSDB_FUNC_BOTTOM 12
#define TSDB_FUNC_SPREAD 13
#define TSDB_FUNC_TWA 14
#define TSDB_FUNC_LEASTSQR 15
#define TSDB_FUNC_TS 16
#define TSDB_FUNC_TS_DUMMY 17
#define TSDB_FUNC_TAG_DUMMY 18
#define TSDB_FUNC_TS_COMP 19
#define TSDB_FUNC_TAG 20
#define TSDB_FUNC_PRJ 21
#define TSDB_FUNC_TAGPRJ 22
#define TSDB_FUNC_ARITHM 23
#define TSDB_FUNC_DIFF 24
#define TSDB_FUNC_FIRST_DST 25
#define TSDB_FUNC_LAST_DST 26
#define TSDB_FUNC_STDDEV_DST 27
#define TSDB_FUNC_INTERP 28
#define TSDB_FUNC_RATE 29
#define TSDB_FUNC_IRATE 30
#define TSDB_FUNC_TID_TAG 31
#define TSDB_FUNC_DERIVATIVE 32
#define TSDB_FUNC_BLKINFO 33
#define TSDB_FUNC_HISTOGRAM 34
#define TSDB_FUNC_HLL 35
#define TSDB_FUNC_MODE 36
#define TSDB_FUNC_SAMPLE 37
#define TSDB_FUNC_CEIL 38
#define TSDB_FUNC_FLOOR 39
#define TSDB_FUNC_ROUND 40
#define TSDB_FUNC_MAVG 41
#define TSDB_FUNC_CSUM 42
#define TSDB_FUNCSTATE_SO 0x1u // single output
#define TSDB_FUNCSTATE_MO 0x2u // dynamic number of output, not multinumber of output e.g., TOP/BOTTOM
#define TSDB_FUNCSTATE_STREAM 0x4u // function avail for stream
#define TSDB_FUNCSTATE_STABLE 0x8u // function avail for super table
#define TSDB_FUNCSTATE_OF 0x10u // outer forward
#define TSDB_FUNCSTATE_NEED_TS 0x20u // timestamp is required during query processing
#define TSDB_FUNCSTATE_SELECTIVITY 0x40u // selectivity functions, can exists along with tag columns
#define TSDB_BASE_FUNC_SO TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_OF
#define TSDB_BASE_FUNC_MO TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_OF
#define TSDB_FUNCTIONS_NAME_MAX_LENGTH 16
#define TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE 50
#define DATA_SET_FLAG ',' // to denote the output area has data, not null value
#define DATA_SET_FLAG_SIZE sizeof(DATA_SET_FLAG)
#define QUERY_ASC_FORWARD_STEP 1
#define QUERY_DESC_FORWARD_STEP -1
#define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP)
#define MAX_INTERVAL_TIME_WINDOW 1000000 // maximum allowed time windows in final results
#define TOP_BOTTOM_QUERY_LIMIT 100
enum {
MASTER_SCAN = 0x0u,
REVERSE_SCAN = 0x1u,
REPEAT_SCAN = 0x2u, //repeat scan belongs to the master scan
MERGE_STAGE = 0x20u,
};
#define QUERY_IS_STABLE_QUERY(type) (((type)&TSDB_QUERY_TYPE_STABLE_QUERY) != 0)
#define QUERY_IS_JOIN_QUERY(type) (TSDB_QUERY_HAS_TYPE(type, TSDB_QUERY_TYPE_JOIN_QUERY))
#define QUERY_IS_PROJECTION_QUERY(type) (((type)&TSDB_QUERY_TYPE_PROJECTION_QUERY) != 0)
#define QUERY_IS_FREE_RESOURCE(type) (((type)&TSDB_QUERY_TYPE_FREE_RESOURCE) != 0)
typedef struct SArithmeticSupport {
SExprInfo *pExprInfo;
int32_t numOfCols;
SColumnInfo *colList;
void *exprList; // client side used
int32_t offset;
char** data;
} SArithmeticSupport;
typedef struct SQLPreAggVal {
bool isSet; // statistics info set or not
bool dataBlockLoaded; // data block is loaded or not
SDataStatis statis;
} SQLPreAggVal;
typedef struct SInterpInfoDetail {
TSKEY ts; // interp specified timestamp
int8_t type;
int8_t primaryCol;
} SInterpInfoDetail;
typedef struct SResultRowCellInfo {
int8_t hasResult; // result generated, not NULL value
bool initialized; // output buffer has been initialized
bool complete; // query has completed
uint32_t numOfRes; // num of output result in current buffer
} SResultRowCellInfo;
typedef struct SPoint1 {
int64_t key;
union{double val; char* ptr;};
} SPoint1;
#define GET_ROWCELL_INTERBUF(_c) ((void*) ((char*)(_c) + sizeof(SResultRowCellInfo)))
struct SQLFunctionCtx;
/**
* for selectivity query, the corresponding tag value is assigned if the data is qualified
*/
typedef struct SExtTagsInfo {
int16_t tagsLen; // keep the tags data for top/bottom query result
int16_t numOfTagCols;
struct SQLFunctionCtx **pTagCtxList;
} SExtTagsInfo;
// sql function runtime context
typedef struct SQLFunctionCtx {
int32_t size; // number of rows
void * pInput; // input data buffer
uint32_t order; // asc|desc
int16_t inputType;
int16_t inputBytes;
int16_t outputType;
int16_t outputBytes; // size of results, determined by function and input column data type
int32_t interBufBytes; // internal buffer size
bool hasNull; // null value exist in current block
bool requireNull; // require null in some function
bool stableQuery;
int16_t functionId; // function id
char * pOutput; // final result output buffer, point to sdata->data
uint8_t currentStage; // record current running step, default: 0
int64_t startTs; // timestamp range of current query when function is executed on a specific data block
int32_t numOfParams;
tVariant param[4]; // input parameter, e.g., top(k, 20), the number of results for top query is kept in param
int64_t *ptsList; // corresponding timestamp array list
void *ptsOutputBuf; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/
SQLPreAggVal preAggVals;
tVariant tag;
SResultRowCellInfo *resultInfo;
SExtTagsInfo tagInfo;
SPoint1 start;
SPoint1 end;
} SQLFunctionCtx;
typedef struct SAggFunctionInfo {
char name[TSDB_FUNCTIONS_NAME_MAX_LENGTH];
uint8_t index; // index of function in aAggs
int8_t stableFuncId; // transfer function for super table query
uint16_t status;
bool (*init)(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultCellInfo); // setup the execute environment
void (*xFunction)(SQLFunctionCtx *pCtx); // blocks version function
// finalizer must be called after all xFunction has been executed to generated final result.
void (*xFinalize)(SQLFunctionCtx *pCtx);
void (*mergeFunc)(SQLFunctionCtx *pCtx);
int32_t (*dataReqFunc)(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId);
} SAggFunctionInfo;
#define GET_RES_INFO(ctx) ((ctx)->resultInfo)
int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, int16_t *type,
int16_t *len, int32_t *interBytes, int16_t extLength, bool isSuperTable, SUdfInfo* pUdfInfo);
int32_t isValidFunction(const char* name, int32_t len);
#define IS_STREAM_QUERY_VALID(x) (((x)&TSDB_FUNCSTATE_STREAM) != 0)
#define IS_MULTIOUTPUT(x) (((x)&TSDB_FUNCSTATE_MO) != 0)
#define IS_SINGLEOUTPUT(x) (((x)&TSDB_FUNCSTATE_SO) != 0)
#define IS_OUTER_FORWARD(x) (((x)&TSDB_FUNCSTATE_OF) != 0)
// determine the real data need to calculated the result
enum {
BLK_DATA_NO_NEEDED = 0x0,
BLK_DATA_STATIS_NEEDED = 0x1,
BLK_DATA_ALL_NEEDED = 0x3,
BLK_DATA_DISCARD = 0x4, // discard current data block since it is not qualified for filter
};
typedef struct STwaInfo {
int8_t hasResult; // flag to denote has value
double dOutput;
SPoint1 p;
STimeWindow win;
} STwaInfo;
struct SBufferWriter;
void blockDistInfoToBinary(STableBlockDist* pDist, struct SBufferWriter* bw);
void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDist* pDist);
/* global sql function array */
extern struct SAggFunctionInfo aAggs[];
extern int32_t functionCompatList[]; // compatible check array list
bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const char *maxval);
/**
* the numOfRes should be kept, since it may be used later
* and allow the ResultInfo to be re initialized
*/
#define RESET_RESULT_INFO(_r) \
do { \
(_r)->initialized = false; \
} while (0)
static FORCE_INLINE void initResultRowEntry(SResultRowCellInfo *pResInfo, int32_t bufLen) {
pResInfo->initialized = true; // the this struct has been initialized flag
pResInfo->complete = false;
pResInfo->hasResult = false;
pResInfo->numOfRes = 0;
memset(GET_ROWCELL_INTERBUF(pResInfo), 0, bufLen);
}
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_QAGGMAIN_H

688
2.0/src/query/inc/qExecutor.h
View File

@ -1,688 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_QEXECUTOR_H
#define TDENGINE_QEXECUTOR_H
#include "os.h"
#include "hash.h"
#include "qAggMain.h"
#include "qFill.h"
#include "qResultbuf.h"
#include "qSqlparser.h"
#include "qTableMeta.h"
#include "qTsbuf.h"
#include "query.h"
#include "taosdef.h"
#include "tarray.h"
#include "tlockfree.h"
#include "tsdb.h"
#include "qUdf.h"
struct SColumnFilterElem;
typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, const char* val1, const char* val2, int16_t type);
typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order);
#define IS_QUERY_KILLED(_q) ((_q)->code == TSDB_CODE_TSC_QUERY_CANCELLED)
#define Q_STATUS_EQUAL(p, s) (((p) & (s)) != 0u)
#define QUERY_IS_ASC_QUERY(q) (GET_FORWARD_DIRECTION_FACTOR((q)->order.order) == QUERY_ASC_FORWARD_STEP)
#define GET_TABLEGROUP(q, _index) ((SArray*) taosArrayGetP((q)->tableqinfoGroupInfo.pGroupList, (_index)))
#define GET_NUM_OF_RESULTS(_r) (((_r)->outputBuf) == NULL? 0:((_r)->outputBuf)->info.rows)
#define NEEDTO_COMPRESS_QUERY(size) ((size) > tsCompressColData? 1 : 0)
enum {
// when query starts to execute, this status will set
QUERY_NOT_COMPLETED = 0x1u,
/* query is over
* 1. this status is used in one row result query process, e.g., count/sum/first/last/ avg...etc.
* 2. when all data within queried time window, it is also denoted as query_completed
*/
QUERY_COMPLETED = 0x2u,
/* when the result is not completed return to client, this status will be
* usually used in case of interval query with interpolation option
*/
QUERY_OVER = 0x4u,
};
typedef struct SResultRowPool {
int32_t elemSize;
int32_t blockSize;
int32_t numOfElemPerBlock;
struct {
int32_t blockIndex;
int32_t pos;
} position;
SArray* pData; // SArray<void*>
} SResultRowPool;
typedef struct SResultRow {
int32_t pageId; // pageId & rowId is the position of current result in disk-based output buffer
int32_t offset:29; // row index in buffer page
bool startInterp; // the time window start timestamp has done the interpolation already.
bool endInterp; // the time window end timestamp has done the interpolation already.
bool closed; // this result status: closed or opened
uint32_t numOfRows; // number of rows of current time window
SResultRowCellInfo* pCellInfo; // For each result column, there is a resultInfo
STimeWindow win;
char *key; // start key of current result row
} SResultRow;
typedef struct SResultRowCell {
uint64_t groupId;
SResultRow *pRow;
} SResultRowCell;
typedef struct SGroupResInfo {
int32_t totalGroup;
int32_t currentGroup;
int32_t index;
SArray* pRows; // SArray<SResultRow*>
bool ordered;
int32_t position;
} SGroupResInfo;
/**
* If the number of generated results is greater than this value,
* query query will be halt and return results to client immediate.
*/
typedef struct SRspResultInfo {
int64_t total; // total generated result size in rows
int32_t capacity; // capacity of current result output buffer
int32_t threshold; // result size threshold in rows.
} SRspResultInfo;
typedef struct SResultRowInfo {
SResultRow** pResult; // result list
int16_t type:8; // data type for hash key
int32_t size:24; // number of result set
int32_t capacity; // max capacity
int32_t curPos; // current active result row index of pResult list
} SResultRowInfo;
typedef struct SColumnFilterElem {
int16_t bytes; // column length
__filter_func_t fp;
SColumnFilterInfo filterInfo;
void *q;
} SColumnFilterElem;
typedef struct SSingleColumnFilterInfo {
void* pData;
void* pData2; //used for nchar column
int32_t numOfFilters;
SColumnInfo info;
SColumnFilterElem* pFilters;
} SSingleColumnFilterInfo;
typedef struct STableQueryInfo {
TSKEY lastKey;
int32_t groupIndex; // group id in table list
tVariant tag;
STimeWindow win;
STSCursor cur;
void* pTable; // for retrieve the page id list
SResultRowInfo resInfo;
} STableQueryInfo;
typedef enum {
QUERY_PROF_BEFORE_OPERATOR_EXEC = 0,
QUERY_PROF_AFTER_OPERATOR_EXEC,
QUERY_PROF_QUERY_ABORT
} EQueryProfEventType;
typedef struct {
EQueryProfEventType eventType;
int64_t eventTime;
union {
uint8_t operatorType; //for operator event
int32_t abortCode; //for query abort event
};
} SQueryProfEvent;
typedef struct {
uint8_t operatorType;
int64_t sumSelfTime;
int64_t sumRunTimes;
} SOperatorProfResult;
typedef struct SQueryCostInfo {
uint64_t loadStatisTime;
uint64_t loadFileBlockTime;
uint64_t loadDataInCacheTime;
uint64_t loadStatisSize;
uint64_t loadFileBlockSize;
uint64_t loadDataInCacheSize;
uint64_t loadDataTime;
uint64_t totalRows;
uint64_t totalCheckedRows;
uint32_t totalBlocks;
uint32_t loadBlocks;
uint32_t loadBlockStatis;
uint32_t skipBlocks;
uint64_t elapsedTime;
uint64_t firstStageMergeTime;
uint64_t winInfoSize;
uint64_t tableInfoSize;
uint64_t hashSize;
uint64_t numOfTimeWindows;
SArray* queryProfEvents; //SArray<SQueryProfEvent>
SHashObj* operatorProfResults; //map<operator_type, SQueryProfEvent>
} SQueryCostInfo;
typedef struct {
int64_t vgroupLimit;
int64_t ts;
} SOrderedPrjQueryInfo;
typedef struct {
char* tags;
SArray* pResult; // SArray<SStddevInterResult>
} SInterResult;
typedef struct SSDataBlock {
SDataStatis *pBlockStatis;
SArray *pDataBlock;
SDataBlockInfo info;
} SSDataBlock;
// The basic query information extracted from the SQueryInfo tree to support the
// execution of query in a data node.
typedef struct SQueryAttr {
SLimitVal limit;
SLimitVal slimit;
bool stableQuery; // super table query or not
bool topBotQuery; // TODO used bitwise flag
bool groupbyColumn; // denote if this is a groupby normal column query
bool hasTagResults; // if there are tag values in final result or not
bool timeWindowInterpo;// if the time window start/end required interpolation
bool queryBlockDist; // if query data block distribution
bool stabledev; // super table stddev query
bool tsCompQuery; // is tscomp query
bool diffQuery; // is diff query
bool simpleAgg;
bool pointInterpQuery; // point interpolation query
bool needReverseScan; // need reverse scan
bool distinct; // distinct query or not
bool stateWindow; // window State on sub/normal table
bool createFilterOperator; // if filter operator is needed
bool multigroupResult; // multigroup result can exist in one SSDataBlock
int32_t interBufSize; // intermediate buffer sizse
int32_t havingNum; // having expr number
SOrderVal order;
int16_t numOfCols;
int16_t numOfTags;
STimeWindow window;
SInterval interval;
SSessionWindow sw;
int16_t precision;
int16_t numOfOutput;
int16_t fillType;
int32_t srcRowSize; // todo extract struct
int32_t resultRowSize;
int32_t intermediateResultRowSize; // intermediate result row size, in case of top-k query.
int32_t maxTableColumnWidth;
int32_t tagLen; // tag value length of current query
SGroupbyExpr *pGroupbyExpr;
SExprInfo* pExpr1;
SExprInfo* pExpr2;
int32_t numOfExpr2;
SExprInfo* pExpr3;
int32_t numOfExpr3;
SColumnInfo* tableCols;
SColumnInfo* tagColList;
int32_t numOfFilterCols;
int64_t* fillVal;
SOrderedPrjQueryInfo prjInfo; // limit value for each vgroup, only available in global order projection query.
SSingleColumnFilterInfo* pFilterInfo;
SFilterInfo *pFilters;
void* tsdb;
SMemRef memRef;
STableGroupInfo tableGroupInfo; // table <tid, last_key> list SArray<STableKeyInfo>
int32_t vgId;
SArray *pUdfInfo; // no need to free
} SQueryAttr;
typedef SSDataBlock* (*__operator_fn_t)(void* param, bool* newgroup);
typedef void (*__optr_cleanup_fn_t)(void* param, int32_t num);
struct SOperatorInfo;
typedef struct SQueryRuntimeEnv {
jmp_buf env;
SQueryAttr* pQueryAttr;
uint32_t status; // query status
void* qinfo;
uint8_t scanFlag; // denotes reversed scan of data or not
void* pTsdbReadHandle;
int32_t prevGroupId; // previous executed group id
bool enableGroupData;
SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file
SHashObj* pResultRowHashTable; // quick locate the window object for each result
SHashObj* pResultRowListSet; // used to check if current ResultRowInfo has ResultRow object or not
SArray* pResultRowArrayList; // The array list that contains the Result rows
char* keyBuf; // window key buffer
SResultRowPool* pool; // The window result objects pool, all the resultRow Objects are allocated and managed by this object.
char** prevRow;
SArray* prevResult; // intermediate result, SArray<SInterResult>
STSBuf* pTsBuf; // timestamp filter list
STSCursor cur;
char* tagVal; // tag value of current data block
SArithmeticSupport *sasArray;
SSDataBlock *outputBuf;
STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray<STableQueryInfo*> structure
struct SOperatorInfo *proot;
SGroupResInfo groupResInfo;
int64_t currentOffset; // dynamic offset value
STableQueryInfo *current;
SRspResultInfo resultInfo;
SHashObj *pTableRetrieveTsMap;
SUdfInfo *pUdfInfo;
} SQueryRuntimeEnv;
enum {
OP_IN_EXECUTING = 1,
OP_RES_TO_RETURN = 2,
OP_EXEC_DONE = 3,
};
enum OPERATOR_TYPE_E {
OP_TableScan = 1,
OP_DataBlocksOptScan = 2,
OP_TableSeqScan = 3,
OP_TagScan = 4,
OP_TableBlockInfoScan= 5,
OP_Aggregate = 6,
OP_Project = 7,
OP_Groupby = 8,
OP_Limit = 9,
OP_SLimit = 10,
OP_TimeWindow = 11,
OP_SessionWindow = 12,
OP_Fill = 13,
OP_MultiTableAggregate = 14,
OP_MultiTableTimeInterval = 15,
OP_DummyInput = 16, //TODO remove it after fully refactor.
OP_MultiwayMergeSort = 17, // multi-way data merge into one input stream.
OP_GlobalAggregate = 18, // global merge for the multi-way data sources.
OP_Filter = 19,
OP_Distinct = 20,
OP_Join = 21,
OP_StateWindow = 22,
OP_AllTimeWindow = 23,
OP_AllMultiTableTimeInterval = 24,
OP_Order = 25,
};
typedef struct SOperatorInfo {
uint8_t operatorType;
bool blockingOptr; // block operator or not
uint8_t status; // denote if current operator is completed
int32_t numOfOutput; // number of columns of the current operator results
char *name; // name, used to show the query execution plan
void *info; // extension attribution
SExprInfo *pExpr;
SQueryRuntimeEnv *pRuntimeEnv;
struct SOperatorInfo **upstream; // upstream pointer list
int32_t numOfUpstream; // number of upstream. The value is always ONE expect for join operator
__operator_fn_t exec;
__optr_cleanup_fn_t cleanup;
} SOperatorInfo;
enum {
QUERY_RESULT_NOT_READY = 1,
QUERY_RESULT_READY = 2,
};
typedef struct {
int32_t numOfTags;
int32_t numOfCols;
SColumnInfo *colList;
} SQueriedTableInfo;
typedef struct SQInfo {
void* signature;
uint64_t qId;
int32_t code; // error code to returned to client
int64_t owner; // if it is in execution
SQueryRuntimeEnv runtimeEnv;
SQueryAttr query;
void* pBuf; // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables;
pthread_mutex_t lock; // used to synchronize the rsp/query threads
tsem_t ready;
int32_t dataReady; // denote if query result is ready or not
void* rspContext; // response context
int64_t startExecTs; // start to exec timestamp
char* sql; // query sql string
SQueryCostInfo summary;
} SQInfo;
typedef struct SQueryParam {
char *sql;
char *tagCond;
char *colCond;
char *tbnameCond;
char *prevResult;
SArray *pTableIdList;
SSqlExpr **pExpr;
SSqlExpr **pSecExpr;
SExprInfo *pExprs;
SExprInfo *pSecExprs;
SFilterInfo *pFilters;
SColIndex *pGroupColIndex;
SColumnInfo *pTagColumnInfo;
SGroupbyExpr *pGroupbyExpr;
int32_t tableScanOperator;
SArray *pOperator;
SUdfInfo *pUdfInfo;
} SQueryParam;
typedef struct STableScanInfo {
void *pTsdbReadHandle;
int32_t numOfBlocks;
int32_t numOfSkipped;
int32_t numOfBlockStatis;
int64_t numOfRows;
int32_t order; // scan order
int32_t times; // repeat counts
int32_t current;
int32_t reverseTimes; // 0 by default
SQLFunctionCtx *pCtx; // next operator query context
SResultRowInfo *pResultRowInfo;
int32_t *rowCellInfoOffset;
SExprInfo *pExpr;
SSDataBlock block;
int32_t numOfOutput;
int64_t elapsedTime;
int32_t tableIndex;
int32_t prevGroupId; // previous table group id
} STableScanInfo;
typedef struct STagScanInfo {
SColumnInfo* pCols;
SSDataBlock* pRes;
int32_t totalTables;
int32_t curPos;
} STagScanInfo;
typedef struct SOptrBasicInfo {
SResultRowInfo resultRowInfo;
int32_t *rowCellInfoOffset; // offset value for each row result cell info
SQLFunctionCtx *pCtx;
SSDataBlock *pRes;
} SOptrBasicInfo;
typedef struct SOptrBasicInfo STableIntervalOperatorInfo;
typedef struct SAggOperatorInfo {
SOptrBasicInfo binfo;
uint32_t seed;
} SAggOperatorInfo;
typedef struct SProjectOperatorInfo {
SOptrBasicInfo binfo;
int32_t bufCapacity;
uint32_t seed;
SSDataBlock *existDataBlock;
} SProjectOperatorInfo;
typedef struct SLimitOperatorInfo {
int64_t limit;
int64_t total;
} SLimitOperatorInfo;
typedef struct SSLimitOperatorInfo {
int64_t groupTotal;
int64_t currentGroupOffset;
int64_t rowsTotal;
int64_t currentOffset;
SLimitVal limit;
SLimitVal slimit;
char **prevRow;
SArray *orderColumnList;
bool hasPrev;
bool ignoreCurrentGroup;
bool multigroupResult;
SSDataBlock *pRes; // result buffer
SSDataBlock *pPrevBlock;
int64_t capacity;
int64_t threshold;
} SSLimitOperatorInfo;
typedef struct SFilterOperatorInfo {
SSingleColumnFilterInfo *pFilterInfo;
int32_t numOfFilterCols;
} SFilterOperatorInfo;
typedef struct SFillOperatorInfo {
SFillInfo *pFillInfo;
SSDataBlock *pRes;
int64_t totalInputRows;
void **p;
SSDataBlock *existNewGroupBlock;
bool multigroupResult;
} SFillOperatorInfo;
typedef struct SGroupbyOperatorInfo {
SOptrBasicInfo binfo;
int32_t colIndex;
char *prevData; // previous group by value
} SGroupbyOperatorInfo;
typedef struct SSWindowOperatorInfo {
SOptrBasicInfo binfo;
STimeWindow curWindow; // current time window
TSKEY prevTs; // previous timestamp
int32_t numOfRows; // number of rows
int32_t start; // start row index
bool reptScan; // next round scan
} SSWindowOperatorInfo;
typedef struct SStateWindowOperatorInfo {
SOptrBasicInfo binfo;
STimeWindow curWindow; // current time window
int32_t numOfRows; // number of rows
int32_t colIndex; // start row index
int32_t start;
char* prevData; // previous data
bool reptScan;
} SStateWindowOperatorInfo;
typedef struct SDistinctDataInfo {
int32_t index;
int32_t type;
int32_t bytes;
} SDistinctDataInfo;
typedef struct SDistinctOperatorInfo {
SHashObj *pSet;
SSDataBlock *pRes;
bool recordNullVal; //has already record the null value, no need to try again
int64_t threshold;
int64_t outputCapacity;
int32_t totalBytes;
char* buf;
SArray* pDistinctDataInfo;
} SDistinctOperatorInfo;
struct SGlobalMerger;
typedef struct SMultiwayMergeInfo {
struct SGlobalMerger *pMerge;
SOptrBasicInfo binfo;
int32_t bufCapacity;
int64_t seed;
char **prevRow;
SArray *orderColumnList;
int32_t resultRowFactor;
bool hasGroupColData;
char **currentGroupColData;
SArray *groupColumnList;
bool hasDataBlockForNewGroup;
SSDataBlock *pExistBlock;
SArray *udfInfo;
bool hasPrev;
bool multiGroupResults;
} SMultiwayMergeInfo;
// todo support the disk-based sort
typedef struct SSortOperatorInfo {
int32_t colIndex;
int32_t order;
SSDataBlock *pDataBlock;
} SSortOperatorInfo;
void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream);
SOperatorInfo* createTableScanOperatorInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime);
SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime);
SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv);
SOperatorInfo* createAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createProjectOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream);
SOperatorInfo* createIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createAllTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createSWindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createFillOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, bool multigroupResult);
SOperatorInfo* createGroupbyOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createMultiTableAggOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createTagScanOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createDistinctOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv);
SOperatorInfo* createMultiwaySortOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput,
int32_t numOfRows, void* merger);
SOperatorInfo* createGlobalAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* param, SArray* pUdfInfo, bool groupResultMixedUp);
SOperatorInfo* createStatewindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createSLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* merger, bool multigroupResult);
SOperatorInfo* createFilterOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr,
int32_t numOfOutput, SColumnInfo* pCols, int32_t numOfFilter);
SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pUpstream, int32_t numOfUpstream, SSchema* pSchema, int32_t numOfOutput);
SOperatorInfo* createSortOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, SOrderVal* pOrderVal);
SSDataBlock* doGlobalAggregate(void* param, bool* newgroup);
SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup);
SSDataBlock* doSLimit(void* param, bool* newgroup);
int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId);
void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, SSDataBlock* pBlock);
bool doFilterDataBlock(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, int32_t numOfRows, int8_t* p);
void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p);
SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numOfRows);
void* blockDataDestroy(SSDataBlock* pBlock);
void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols);
void setInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order);
int32_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput);
void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset);
void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOfInputRows);
void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity);
void copyTsColoum(SSDataBlock* pRes, SQLFunctionCtx* pCtx, int32_t numOfOutput);
void freeParam(SQueryParam *param);
int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param);
int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExprInfo** pExprInfo,
SSqlExpr** pExprMsg, SColumnInfo* pTagCols, int32_t queryType, void* pMsg, SUdfInfo* pUdfInfo);
int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t numOfOutput, SExprInfo **pExprInfo,
SSqlExpr **pExpr, SExprInfo *prevExpr, SUdfInfo *pUdfInfo);
int32_t createQueryFilter(char *data, uint16_t len, SFilterInfo** pFilters);
SGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pColIndex, int32_t *code);
SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs,
SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, SFilterInfo* pFilters, int32_t vgId, char* sql, uint64_t qId, SUdfInfo* pUdfInfo);
int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* pQInfo, SQueryParam* param, char* start,
int32_t prevResultLen, void* merger);
int32_t createFilterInfo(SQueryAttr* pQueryAttr, uint64_t qId);
void freeColumnFilterInfo(SColumnFilterInfo* pFilter, int32_t numOfFilters);
STableQueryInfo *createTableQueryInfo(SQueryAttr* pQueryAttr, void* pTable, bool groupbyColumn, STimeWindow win, void* buf);
STableQueryInfo* createTmpTableQueryInfo(STimeWindow win);
int32_t buildArithmeticExprFromMsg(SExprInfo *pArithExprInfo, void *pQueryMsg);
bool isQueryKilled(SQInfo *pQInfo);
int32_t checkForQueryBuf(size_t numOfTables);
bool checkNeedToCompressQueryCol(SQInfo *pQInfo);
bool doBuildResCheck(SQInfo* pQInfo);
void setQueryStatus(SQueryRuntimeEnv *pRuntimeEnv, int8_t status);
bool onlyQueryTags(SQueryAttr* pQueryAttr);
void destroyUdfInfo(SUdfInfo* pUdfInfo);
bool isValidQInfo(void *param);
int32_t doDumpQueryResult(SQInfo *pQInfo, char *data, int8_t compressed, int32_t *compLen);
size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows);
void setQueryKilled(SQInfo *pQInfo);
void publishOperatorProfEvent(SOperatorInfo* operatorInfo, EQueryProfEventType eventType);
void publishQueryAbortEvent(SQInfo* pQInfo, int32_t code);
void calculateOperatorProfResults(SQInfo* pQInfo);
void queryCostStatis(SQInfo *pQInfo);
void freeQInfo(SQInfo *pQInfo);
void freeQueryAttr(SQueryAttr *pQuery);
int32_t getMaximumIdleDurationSec();
void doInvokeUdf(SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, int32_t type);
#endif // TDENGINE_QEXECUTOR_H

253
2.0/src/query/inc/qExtbuffer.h
View File

@ -1,253 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TEXTBUFFER_H
#define TDENGINE_TEXTBUFFER_H
#ifdef __cplusplus
extern "C" {
#endif
#include "os.h"
#include "tmsg.h"
#include "tarray.h"
#include "tutil.h"
#include "tdataformat.h"
#include "talgo.h"
#define MAX_TMPFILE_PATH_LENGTH PATH_MAX
#define INITIAL_ALLOCATION_BUFFER_SIZE 64
#define DEFAULT_PAGE_SIZE (1024L) // 16k larger than the SHistoInfo
typedef enum EXT_BUFFER_FLUSH_MODEL {
/*
* all data that have been flushed to disk is belonged to the same group
* which means, all data in disk are sorted, or order is not matter in this case
*/
SINGLE_APPEND_MODEL,
/*
* each flush operation to disk is completely independant to any other flush operation
* we simply merge several set of data in one file, to reduce the count of flat files
* in disk. So in this case, we need to keep the flush-out information in tFlushoutInfo
* structure.
*/
MULTIPLE_APPEND_MODEL,
} EXT_BUFFER_FLUSH_MODEL;
typedef struct tFlushoutInfo {
uint32_t startPageId;
uint32_t numOfPages;
} tFlushoutInfo;
typedef struct tFlushoutData {
uint32_t nAllocSize;
uint32_t nLength;
tFlushoutInfo *pFlushoutInfo;
} tFlushoutData;
typedef struct SExtFileInfo {
uint32_t nFileSize; // in pages
uint32_t pageSize;
uint32_t numOfElemsInFile;
tFlushoutData flushoutData;
} SExtFileInfo;
typedef struct tFilePage {
uint64_t num;
char data[];
} tFilePage;
typedef struct tFilePagesItem {
struct tFilePagesItem *pNext;
tFilePage item;
} tFilePagesItem;
typedef struct SSchemaEx {
struct SSchema field;
int32_t offset;
} SSchemaEx;
typedef struct SColumnModel {
int32_t capacity;
int32_t numOfCols;
int32_t rowSize;
SSchemaEx *pFields;
} SColumnModel;
typedef struct SColumnOrderInfo {
int32_t numOfCols;
int16_t colIndex[];
} SColumnOrderInfo;
typedef struct tOrderDescriptor {
SColumnModel * pColumnModel;
int32_t tsOrder; // timestamp order type if exists
SColumnOrderInfo orderInfo;
} tOrderDescriptor;
typedef struct tExtMemBuffer {
int32_t inMemCapacity;
int32_t nElemSize;
int32_t pageSize;
int32_t numOfTotalElems;
int32_t numOfElemsInBuffer;
int32_t numOfElemsPerPage;
int16_t numOfInMemPages;
tFilePagesItem *pHead;
tFilePagesItem *pTail;
char * path;
FILE * file;
SExtFileInfo fileMeta;
SColumnModel * pColumnModel;
EXT_BUFFER_FLUSH_MODEL flushModel;
} tExtMemBuffer;
/**
*
* @param inMemSize
* @param elemSize
* @param pModel
* @return
*/
tExtMemBuffer *createExtMemBuffer(int32_t inMemSize, int32_t elemSize, int32_t pagesize, SColumnModel *pModel);
/**
*
* @param pMemBuffer
* @return
*/
void *destoryExtMemBuffer(tExtMemBuffer *pMemBuffer);
/**
* @param pMemBuffer
* @param data input data pointer
* @param numOfRows number of rows in data
* @param pModel column format model
* @return number of pages in memory
*/
int16_t tExtMemBufferPut(tExtMemBuffer *pMemBuffer, void *data, int32_t numOfRows);
/**
*
* @param pMemBuffer
* @return
*/
int32_t tExtMemBufferFlush(tExtMemBuffer *pMemBuffer);
/**
*
* remove all data that has been put into buffer, including in buffer or
* ext-buffer(disk)
*/
void tExtMemBufferClear(tExtMemBuffer *pMemBuffer);
/*
* this function should be removed.
* since the flush to disk operation is transparent to client this structure should provide stream operation for data,
* and there is an internal cursor point to the data.
*/
bool tExtMemBufferLoadData(tExtMemBuffer *pMemBuffer, tFilePage *pFilePage, int32_t flushIdx, int32_t pageIdx);
/**
*
* @param pMemBuffer
* @return
*/
bool tExtMemBufferIsAllDataInMem(tExtMemBuffer *pMemBuffer);
/**
*
* @param fields
* @param numOfCols
* @param blockCapacity
* @return
*/
SColumnModel *createColumnModel(SSchema *fields, int32_t numOfCols, int32_t blockCapacity);
/**
*
* @param pSrc
* @return
*/
SColumnModel *cloneColumnModel(SColumnModel *pSrc);
/**
*
* @param pModel
*/
void destroyColumnModel(SColumnModel *pModel);
/*
* compress data into consecutive block without hole in data
*/
void tColModelCompact(SColumnModel *pModel, tFilePage *inputBuffer, int32_t maxElemsCapacity);
void tColModelErase(SColumnModel *pModel, tFilePage *inputBuffer, int32_t maxCapacity, int32_t s, int32_t e);
SSchema *getColumnModelSchema(SColumnModel *pColumnModel, int32_t index);
int16_t getColumnModelOffset(SColumnModel *pColumnModel, int32_t index);
typedef struct SSrcColumnInfo {
int32_t functionId;
int32_t type;
} SSrcColumnInfo;
/*
* display data in column format model for debug purpose only
*/
void tColModelDisplay(SColumnModel *pModel, void *pData, int32_t numOfRows, int32_t maxCount);
void tColModelDisplayEx(SColumnModel *pModel, void *pData, int32_t numOfRows, int32_t maxCount, SSrcColumnInfo *pInfo);
tOrderDescriptor *tOrderDesCreate(const int32_t *orderColIdx, int32_t numOfOrderCols, SColumnModel *pModel,
int32_t tsOrderType);
void tOrderDescDestroy(tOrderDescriptor *pDesc);
void taoscQSort(void** pCols, SSchema* pSchema, int32_t numOfCols, int32_t numOfRows, int32_t index, __compar_fn_t compareFn);
void tColModelAppend(SColumnModel *dstModel, tFilePage *dstPage, void *srcData, int32_t srcStartRows,
int32_t numOfRowsToWrite, int32_t srcCapacity);
typedef int (*__col_compar_fn_t)(tOrderDescriptor *, int32_t numOfRows, int32_t idx1, int32_t idx2, char *data);
void tColDataQSort(tOrderDescriptor *, int32_t numOfRows, int32_t start, int32_t end, char *data, int32_t orderType);
void taoscQSort(void** pCols, SSchema* pSchema, int32_t numOfCols, int32_t numOfRows, int32_t index, __compar_fn_t compareFn);
int32_t compare_sa(tOrderDescriptor *, int32_t numOfRows, int32_t idx1, int32_t idx2, char *data);
int32_t compare_sd(tOrderDescriptor *, int32_t numOfRows, int32_t idx1, int32_t idx2, char *data);
int32_t compare_a(tOrderDescriptor *, int32_t numOfRow1, int32_t s1, char *data1, int32_t numOfRow2, int32_t s2,
char *data2);
int32_t compare_d(tOrderDescriptor *, int32_t numOfRow1, int32_t s1, char *data1, int32_t numOfRow2, int32_t s2,
char *data2);
struct SSDataBlock;
int32_t compare_aRv(struct SSDataBlock* pBlock, SArray* colIndex, int32_t numOfCols, int32_t rowIndex, char** buffer, int32_t order);
int32_t columnValueAscendingComparator(char *f1, char *f2, int32_t type, int32_t bytes);
#ifdef __cplusplus
}
#endif
#endif // TBASE_SORT_H

97
2.0/src/query/inc/qFill.h
View File

@ -1,97 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_QFILL_H
#define TDENGINE_QFILL_H
#ifdef __cplusplus
extern "C" {
#endif
#include "os.h"
#include "qExtbuffer.h"
#include "taosdef.h"
struct SSDataBlock;
typedef struct {
STColumn col; // column info
int16_t functionId; // sql function id
int16_t flag; // column flag: TAG COLUMN|NORMAL COLUMN
int16_t tagIndex; // index of current tag in SFillTagColInfo array list
union {int64_t i; double d;} fillVal;
} SFillColInfo;
typedef struct {
SSchema col;
char* tagVal;
} SFillTagColInfo;
typedef struct SFillInfo {
TSKEY start; // start timestamp
TSKEY end; // endKey for fill
TSKEY currentKey; // current active timestamp, the value may be changed during the fill procedure.
int32_t order; // order [TSDB_ORDER_ASC|TSDB_ORDER_DESC]
int32_t type; // fill type
int32_t numOfRows; // number of rows in the input data block
int32_t index; // active row index
int32_t numOfTotal; // number of filled rows in one round
int32_t numOfCurrent; // number of filled rows in current results
int32_t numOfTags; // number of tags
int32_t numOfCols; // number of columns, including the tags columns
int32_t rowSize; // size of each row
SInterval interval;
char * prevValues; // previous row of data, to generate the interpolation results
char * nextValues; // next row of data
char** pData; // original result data block involved in filling data
int32_t alloc; // data buffer size in rows
int8_t precision; // time resoluation
SFillColInfo* pFillCol; // column info for fill operations
SFillTagColInfo* pTags; // tags value for filling gap
void* handle; // for debug purpose
} SFillInfo;
typedef struct SPoint {
int64_t key;
void * val;
} SPoint;
SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols,
int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType,
SFillColInfo* pFillCol, void* handle);
void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp);
void* taosDestroyFillInfo(SFillInfo *pFillInfo);
void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey);
void taosFillSetInputDataBlock(SFillInfo* pFillInfo, const struct SSDataBlock* pInput);
bool taosFillHasMoreResults(SFillInfo* pFillInfo);
int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows);
int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2, int32_t inputType);
int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, void** output, int32_t capacity);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_QFILL_H

79
2.0/src/query/inc/qHistogram.h
View File

@ -1,79 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_HISTOGRAM_H
#define TDENGINE_HISTOGRAM_H
#ifdef __cplusplus
extern "C" {
#endif
#define USE_ARRAYLIST
#define MAX_HISTOGRAM_BIN 500
typedef struct SHistBin {
double val;
int64_t num;
#if !defined(USE_ARRAYLIST)
double delta;
int32_t index; // index in min-heap list
#endif
} SHistBin;
typedef struct SHeapEntry {
void* pData;
double val;
} SHeapEntry;
typedef struct SHistogramInfo {
int64_t numOfElems;
int32_t numOfEntries;
int32_t maxEntries;
double min;
double max;
#if defined(USE_ARRAYLIST)
SHistBin* elems;
#else
tSkipList* pList;
SLoserTreeInfo* pLoserTree;
int32_t maxIndex;
bool ordered;
#endif
} SHistogramInfo;
SHistogramInfo* tHistogramCreate(int32_t numOfBins);
SHistogramInfo* tHistogramCreateFrom(void* pBuf, int32_t numOfBins);
int32_t tHistogramAdd(SHistogramInfo** pHisto, double val);
int64_t tHistogramSum(SHistogramInfo* pHisto, double v);
double* tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num);
SHistogramInfo* tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2, int32_t numOfEntries);
void tHistogramDestroy(SHistogramInfo** pHisto);
void tHistogramPrint(SHistogramInfo* pHisto);
int32_t histoBinarySearch(SHistBin* pEntry, int32_t len, double val);
SHeapEntry* tHeapCreate(int32_t numOfEntries);
void tHeapSort(SHeapEntry* pEntry, int32_t len);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_HISTOGRAM_H

83
2.0/src/query/inc/qPercentile.h
View File

@ -1,83 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_QPERCENTILE_H
#define TDENGINE_QPERCENTILE_H
#ifdef __cplusplus
extern "C" {
#endif
#include "qExtbuffer.h"
#include "qResultbuf.h"
#include "qTsbuf.h"
typedef struct MinMaxEntry {
union {
double dMinVal;
int64_t i64MinVal;
uint64_t u64MinVal;
};
union {
double dMaxVal;
int64_t i64MaxVal;
int64_t u64MaxVal;
};
} MinMaxEntry;
typedef struct {
int32_t size;
int32_t pageId;
tFilePage *data;
} SSlotInfo;
typedef struct tMemBucketSlot {
SSlotInfo info;
MinMaxEntry range;
} tMemBucketSlot;
struct tMemBucket;
typedef int32_t (*__perc_hash_func_t)(struct tMemBucket *pBucket, const void *value);
typedef struct tMemBucket {
int16_t numOfSlots;
int16_t type;
int16_t bytes;
int32_t total;
int32_t elemPerPage; // number of elements for each object
int32_t maxCapacity; // maximum allowed number of elements that can be sort directly to get the result
int32_t bufPageSize; // disk page size
MinMaxEntry range; // value range
int32_t times; // count that has been checked for deciding the correct data value buckets.
__compar_fn_t comparFn;
tMemBucketSlot * pSlots;
SDiskbasedResultBuf *pBuffer;
__perc_hash_func_t hashFunc;
} tMemBucket;
tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, double maxval);
void tMemBucketDestroy(tMemBucket *pBucket);
int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size);
double getPercentile(tMemBucket *pMemBucket, double percent);
#endif // TDENGINE_QPERCENTILE_H
#ifdef __cplusplus
}
#endif

57
2.0/src/query/inc/qPlan.h
View File

@ -1,57 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_QPLAN_H
#define TDENGINE_QPLAN_H
#include "qExecutor.h"
struct SQueryInfo;
typedef struct SQueryNodeBasicInfo {
int32_t type;
char *name;
} SQueryNodeBasicInfo;
typedef struct SQueryTableInfo {
char *tableName;
STableId id;
} SQueryTableInfo;
typedef struct SQueryNode {
SQueryNodeBasicInfo info;
SQueryTableInfo tableInfo;
SSchema *pSchema; // the schema of the input SSDatablock
int32_t numOfCols; // number of input columns
SExprInfo *pExpr; // the query functions or sql aggregations
int32_t numOfOutput; // number of result columns, which is also the number of pExprs
void *pExtInfo; // additional information
// previous operator to generated result for current node to process
// in case of join, multiple prev nodes exist.
SArray *pPrevNodes;// upstream nodes
struct SQueryNode *nextNode;
} SQueryNode;
SQueryNode* qCreateQueryPlan(struct SQueryInfo* pQueryInfo);
void* qDestroyQueryPlan(SQueryNode* pQueryNode);
char* queryPlanToString(SQueryNode* pQueryNode);
SArray* createTableScanPlan(SQueryAttr* pQueryAttr);
SArray* createExecOperatorPlan(SQueryAttr* pQueryAttr);
SArray* createGlobalMergePlan(SQueryAttr* pQueryAttr);
#endif // TDENGINE_QPLAN_H

164
2.0/src/query/inc/qResultbuf.h
View File

@ -1,164 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_QRESULTBUF_H
#define TDENGINE_QRESULTBUF_H
#ifdef __cplusplus
extern "C" {
#endif
#include <tlist.h>
#include "hash.h"
#include "os.h"
#include "qExtbuffer.h"
#include "tlockfree.h"
typedef struct SArray* SIDList;
typedef struct SPageDiskInfo {
int32_t offset;
int32_t length;
} SPageDiskInfo;
typedef struct SPageInfo {
SListNode* pn; // point to list node
int32_t pageId;
SPageDiskInfo info;
void* pData;
bool used; // set current page is in used
} SPageInfo;
typedef struct SFreeListItem {
int32_t offset;
int32_t len;
} SFreeListItem;
typedef struct SResultBufStatis {
int32_t flushBytes;
int32_t loadBytes;
int32_t getPages;
int32_t releasePages;
int32_t flushPages;
} SResultBufStatis;
typedef struct SDiskbasedResultBuf {
int32_t numOfPages;
int64_t totalBufSize;
int64_t fileSize; // disk file size
FILE* file;
int32_t allocateId; // allocated page id
char* path; // file path
int32_t pageSize; // current used page size
int32_t inMemPages; // numOfPages that are allocated in memory
SHashObj* groupSet; // id hash table
SHashObj* all;
SList* lruList;
void* emptyDummyIdList; // dummy id list
void* assistBuf; // assistant buffer for compress/decompress data
SArray* pFree; // free area in file
bool comp; // compressed before flushed to disk
int32_t nextPos; // next page flush position
uint64_t qId; // for debug purpose
SResultBufStatis statis;
} SDiskbasedResultBuf;
#define DEFAULT_INTERN_BUF_PAGE_SIZE (1024L) // in bytes
#define PAGE_INFO_INITIALIZER (SPageDiskInfo){-1, -1}
/**
* create disk-based result buffer
* @param pResultBuf
* @param rowSize
* @param pagesize
* @param inMemPages
* @param handle
* @return
*/
int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId);
/**
*
* @param pResultBuf
* @param groupId
* @param pageId
* @return
*/
tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId);
/**
*
* @param pResultBuf
* @param groupId
* @return
*/
SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId);
/**
* get the specified buffer page by id
* @param pResultBuf
* @param id
* @return
*/
tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id);
/**
* release the referenced buf pages
* @param pResultBuf
* @param page
*/
void releaseResBufPage(SDiskbasedResultBuf* pResultBuf, void* page);
/**
*
* @param pResultBuf
* @param pi
*/
void releaseResBufPageInfo(SDiskbasedResultBuf* pResultBuf, SPageInfo* pi);
/**
* get the total buffer size in the format of disk file
* @param pResultBuf
* @return
*/
size_t getResBufSize(const SDiskbasedResultBuf* pResultBuf);
/**
* get the number of groups in the result buffer
* @param pResultBuf
* @return
*/
size_t getNumOfResultBufGroupId(const SDiskbasedResultBuf* pResultBuf);
/**
* destroy result buffer
* @param pResultBuf
*/
void destroyResultBuf(SDiskbasedResultBuf* pResultBuf);
/**
*
* @param pList
* @return
*/
SPageInfo* getLastPageInfo(SIDList pList);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_QRESULTBUF_H

374
2.0/src/query/inc/qSqlparser.h
View File

@ -1,374 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_QSQLPARSER_H
#define TDENGINE_QSQLPARSER_H
#ifdef __cplusplus
extern "C" {
#endif
#include "../../../include/client/taos.h"
#include "tmsg.h"
#include "tstrbuild.h"
#include "ttoken.h"
#include "tvariant.h"
#define ParseTOKENTYPE SStrToken
#define NON_ARITHMEIC_EXPR 0
#define NORMAL_ARITHMETIC 1
#define AGG_ARIGHTMEIC 2
enum SQL_NODE_TYPE {
SQL_NODE_TABLE_COLUMN= 1,
SQL_NODE_SQLFUNCTION = 2,
SQL_NODE_VALUE = 3,
SQL_NODE_EXPR = 4,
};
enum SQL_NODE_FROM_TYPE {
SQL_FROM_NODE_SUBQUERY = 1,
SQL_FROM_NODE_TABLES = 2,
};
enum SQL_EXPR_FLAG {
EXPR_FLAG_TS_ERROR = 1,
EXPR_FLAG_NS_TIMESTAMP = 2,
EXPR_FLAG_TIMESTAMP_VAR = 3,
};
extern char tTokenTypeSwitcher[13];
#define toTSDBType(x) \
do { \
if ((x) >= tListLen(tTokenTypeSwitcher)) { \
(x) = TSDB_DATA_TYPE_BINARY; \
} else { \
(x) = tTokenTypeSwitcher[(x)]; \
} \
} while (0)
#define TPARSER_HAS_TOKEN(_t) ((_t).n > 0)
#define TPARSER_SET_NONE_TOKEN(_t) ((_t).n = 0)
typedef struct SLimitVal {
int64_t limit;
int64_t offset;
} SLimitVal;
typedef struct SOrderVal {
uint32_t order;
int32_t orderColId;
} SOrderVal;
typedef struct tVariantListItem {
tVariant pVar;
uint8_t sortOrder;
} tVariantListItem;
typedef struct SIntervalVal {
int32_t token;
SStrToken interval;
SStrToken offset;
} SIntervalVal;
typedef struct SSessionWindowVal {
SStrToken col;
SStrToken gap;
} SSessionWindowVal;
typedef struct SWindowStateVal {
SStrToken col;
} SWindowStateVal;
struct SRelationInfo;
typedef struct SSqlNode {
struct SArray *pSelNodeList; // select clause
struct SRelationInfo *from; // from clause SArray<SSqlNode>
struct tSqlExpr *pWhere; // where clause [optional]
SArray *pGroupby; // groupby clause, only for tags[optional], SArray<tVariantListItem>
SArray *pSortOrder; // orderby [optional], SArray<tVariantListItem>
SArray *fillType; // fill type[optional], SArray<tVariantListItem>
SIntervalVal interval; // (interval, interval_offset) [optional]
SSessionWindowVal sessionVal; // session window [optional]
SWindowStateVal windowstateVal; // window_state(col) [optional]
SStrToken sliding; // sliding window [optional]
SLimitVal limit; // limit offset [optional]
SLimitVal slimit; // group limit offset [optional]
SStrToken sqlstr; // sql string in select clause
struct tSqlExpr *pHaving; // having clause [optional]
} SSqlNode;
typedef struct SRelElement {
union {
SStrToken tableName;
SArray *pSubquery;
};
SStrToken aliasName;
} SRelElement;
typedef struct SRelationInfo {
int32_t type; // nested query|table name list
SArray *list; // SArray<SRelElement>
} SRelationInfo;
typedef struct SCreatedTableInfo {
SStrToken name; // table name token
SStrToken stableName; // super table name token , for using clause
SArray *pTagNames; // create by using super table, tag name
SArray *pTagVals; // create by using super table, tag value
char *fullname; // table full name
STagData tagdata; // true tag data, super table full name is in STagData
int8_t igExist; // ignore if exists
} SCreatedTableInfo;
typedef struct SCreateTableSql {
SStrToken name; // table name, create table [name] xxx
int8_t type; // create normal table/from super table/ stream
bool existCheck;
struct {
SArray *pTagColumns; // SArray<TAOS_FIELD>
SArray *pColumns; // SArray<TAOS_FIELD>
} colInfo;
SArray *childTableInfo; // SArray<SCreatedTableInfo>
SSqlNode *pSelect;
} SCreateTableSql;
typedef struct SAlterTableInfo {
SStrToken name;
int16_t tableType;
int16_t type;
STagData tagData;
SArray *pAddColumns; // SArray<TAOS_FIELD>
SArray *varList; // set t=val or: change src dst, SArray<tVariantListItem>
} SAlterTableInfo;
typedef struct SCreateDbInfo {
SStrToken dbname;
int32_t replica;
int32_t cacheBlockSize;
int32_t maxTablesPerVnode;
int32_t numOfBlocks;
int32_t daysPerFile;
int32_t minRowsPerBlock;
int32_t maxRowsPerBlock;
int32_t fsyncPeriod;
int64_t commitTime;
int32_t walLevel;
int32_t quorum;
int32_t compressionLevel;
SStrToken precision;
bool ignoreExists;
int8_t update;
int8_t cachelast;
SArray *keep;
int8_t dbType;
int16_t partitions;
} SCreateDbInfo;
typedef struct SCreateFuncInfo {
SStrToken name;
SStrToken path;
int32_t type;
int32_t bufSize;
TAOS_FIELD output;
} SCreateFuncInfo;
typedef struct SCreateAcctInfo {
int32_t maxUsers;
int32_t maxDbs;
int32_t maxTimeSeries;
int32_t maxStreams;
int32_t maxPointsPerSecond;
int64_t maxStorage;
int64_t maxQueryTime;
int32_t maxConnections;
SStrToken stat;
} SCreateAcctInfo;
typedef struct SShowInfo {
uint8_t showType;
SStrToken prefix;
SStrToken pattern;
} SShowInfo;
typedef struct SUserInfo {
SStrToken user;
SStrToken passwd;
SStrToken privilege;
int16_t type;
} SUserInfo;
typedef struct SMiscInfo {
SArray *a; // SArray<SStrToken>
bool existsCheck;
int16_t dbType;
int16_t tableType;
SUserInfo user;
union {
SCreateDbInfo dbOpt;
SCreateAcctInfo acctOpt;
SCreateFuncInfo funcOpt;
SShowInfo showOpt;
SStrToken id;
};
} SMiscInfo;
typedef struct SSqlInfo {
int32_t type;
bool valid;
SArray *list; // todo refactor
char msg[256];
SArray *funcs;
union {
SCreateTableSql *pCreateTableInfo;
SAlterTableInfo *pAlterInfo;
SMiscInfo *pMiscInfo;
};
} SSqlInfo;
typedef struct tSqlExpr {
uint16_t type; // sql node type
uint32_t tokenId; // TK_LE: less than(binary expr)
// the whole string of the function(col, param), while the function name is kept in exprToken
struct {
SStrToken operand;
struct SArray *paramList; // function parameters list
} Expr;
int32_t functionId; // function id, todo remove it
SStrToken columnName; // table column info
tVariant value; // the use input value
SStrToken exprToken; // original sql expr string
uint32_t flags; // todo remove it
struct tSqlExpr *pLeft; // left child
struct tSqlExpr *pRight; // right child
} tSqlExpr;
// used in select clause. select <SArray> from xxx
typedef struct tSqlExprItem {
tSqlExpr *pNode; // The list of expressions
char *aliasName; // alias name, null-terminated string
bool distinct;
} tSqlExprItem;
SArray *tVariantListAppend(SArray *pList, tVariant *pVar, uint8_t sortOrder);
SArray *tVariantListInsert(SArray *pList, tVariant *pVar, uint8_t sortOrder, int32_t index);
SArray *tVariantListAppendToken(SArray *pList, SStrToken *pAliasToken, uint8_t sortOrder);
SRelationInfo *setTableNameList(SRelationInfo* pFromInfo, SStrToken *pName, SStrToken* pAlias);
void *destroyRelationInfo(SRelationInfo* pFromInfo);
SRelationInfo *addSubqueryElem(SRelationInfo* pRelationInfo, SArray* pSub, SStrToken* pAlias);
// sql expr leaf node
tSqlExpr *tSqlExprCreateIdValue(SStrToken *pToken, int32_t optrType);
tSqlExpr *tSqlExprCreateFunction(SArray *pParam, SStrToken *pFuncToken, SStrToken *endToken, int32_t optType);
SArray *tStrTokenAppend(SArray *pList, SStrToken *pToken);
tSqlExpr *tSqlExprCreate(tSqlExpr *pLeft, tSqlExpr *pRight, int32_t optrType);
tSqlExpr *tSqlExprClone(tSqlExpr *pSrc);
void tSqlExprCompact(tSqlExpr** pExpr);
bool tSqlExprIsLeaf(tSqlExpr* pExpr);
bool tSqlExprIsParentOfLeaf(tSqlExpr* pExpr);
void tSqlExprDestroy(tSqlExpr *pExpr);
SArray *tSqlExprListAppend(SArray *pList, tSqlExpr *pNode, SStrToken *pDistinct, SStrToken *pToken);
void tSqlExprListDestroy(SArray *pList);
SSqlNode *tSetQuerySqlNode(SStrToken *pSelectToken, SArray *pSelNodeList, SRelationInfo *pFrom, tSqlExpr *pWhere,
SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval, SSessionWindowVal *ps, SWindowStateVal *pw,
SStrToken *pSliding, SArray *pFill, SLimitVal *pLimit, SLimitVal *pgLimit, tSqlExpr *pHaving);
int32_t tSqlExprCompare(tSqlExpr *left, tSqlExpr *right);
SCreateTableSql *tSetCreateTableInfo(SArray *pCols, SArray *pTags, SSqlNode *pSelect, int32_t type);
SAlterTableInfo *tSetAlterTableInfo(SStrToken *pTableName, SArray *pCols, SArray *pVals, int32_t type, int16_t tableTable);
SCreatedTableInfo createNewChildTableInfo(SStrToken *pTableName, SArray *pTagNames, SArray *pTagVals, SStrToken *pToken, SStrToken* igExists);
void destroyAllSqlNode(SArray *pSqlNode);
void destroySqlNode(SSqlNode *pSql);
void freeCreateTableInfo(void* p);
SSqlInfo *setSqlInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SStrToken *pTableName, int32_t type);
SArray *setSubclause(SArray *pList, void *pSqlNode);
SArray *appendSelectClause(SArray *pList, void *pSubclause);
void setCreatedTableName(SSqlInfo *pInfo, SStrToken *pTableNameToken, SStrToken *pIfNotExists);
void SqlInfoDestroy(SSqlInfo *pInfo);
void setDCLSqlElems(SSqlInfo *pInfo, int32_t type, int32_t nParams, ...);
void setDropDbTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrToken* existsCheck,int16_t dbType,int16_t tableType);
void setShowOptions(SSqlInfo *pInfo, int32_t type, SStrToken* prefix, SStrToken* pPatterns);
void setCreateDbInfo(SSqlInfo *pInfo, int32_t type, SStrToken *pToken, SCreateDbInfo *pDB, SStrToken *pIgExists);
void setCreateAcctSql(SSqlInfo *pInfo, int32_t type, SStrToken *pName, SStrToken *pPwd, SCreateAcctInfo *pAcctInfo);
void setCreateUserSql(SSqlInfo *pInfo, SStrToken *pName, SStrToken *pPasswd);
void setKillSql(SSqlInfo *pInfo, int32_t type, SStrToken *ip);
void setAlterUserSql(SSqlInfo *pInfo, int16_t type, SStrToken *pName, SStrToken* pPwd, SStrToken *pPrivilege);
void setCompactVnodeSql(SSqlInfo *pInfo, int32_t type, SArray *pParam);
void setDefaultCreateDbOption(SCreateDbInfo *pDBInfo);
void setDefaultCreateTopicOption(SCreateDbInfo *pDBInfo);
// prefix show db.tables;
void tSetDbName(SStrToken *pCpxName, SStrToken *pDb);
void tSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType);
void tSetColumnType(TAOS_FIELD *pField, SStrToken *type);
/**
*
* @param yyp The parser
* @param yymajor The major token code number
* @param yyminor The value for the token
*/
void Parse(void *yyp, int yymajor, ParseTOKENTYPE yyminor, SSqlInfo *);
/**
*
* @param p The parser to be deleted
* @param freeProc Function used to reclaim memory
*/
void ParseFree(void *p, void (*freeProc)(void *));
/**
*
* @param mallocProc The parser allocator
* @return
*/
void *ParseAlloc(void *(*mallocProc)(size_t));
/**
*
* @param str sql string
* @return sql ast
*/
SSqlInfo qSqlParse(const char *str);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_QSQLPARSER_H

230
2.0/src/query/inc/qTableMeta.h
View File

@ -1,230 +0,0 @@
#ifndef TDENGINE_QTABLEUTIL_H
#define TDENGINE_QTABLEUTIL_H
#include "qSqlparser.h"
#include "tfilter.h"
#include "tsdb.h" //todo tsdb should not be here
typedef struct SFieldInfo {
int16_t numOfOutput; // number of column in result
TAOS_FIELD* final;
SArray *internalField; // SArray<SInternalField>
} SFieldInfo;
typedef struct SCond {
uint64_t uid;
int32_t len; // length of tag query condition data
char * cond;
} SCond;
typedef struct STblCond {
uint64_t uid;
int16_t idx; //table index
int32_t len; // length of tag query condition data
char * cond;
} STblCond;
typedef struct SJoinNode {
uint64_t uid;
int16_t tagColId;
SArray* tsJoin;
SArray* tagJoin;
} SJoinNode;
typedef struct SJoinInfo {
bool hasJoin;
SJoinNode *joinTables[TSDB_MAX_JOIN_TABLE_NUM];
} SJoinInfo;
typedef struct STagCond {
// relation between tbname list and query condition, including : TK_AND or TK_OR
int16_t relType;
// tbname query condition, only support tbname query condition on one table
SCond tbnameCond;
// join condition, only support two tables join currently
SJoinInfo joinInfo;
// for different table, the query condition must be seperated
SArray *pCond;
} STagCond;
typedef struct SGroupbyExpr {
int16_t tableIndex;
SArray* columnInfo; // SArray<SColIndex>, group by columns information
int16_t numOfGroupCols; // todo remove it
int16_t orderIndex; // order by column index
int16_t orderType; // order by type: asc/desc
} SGroupbyExpr;
typedef struct STableComInfo {
uint8_t numOfTags;
uint8_t precision;
int16_t numOfColumns;
int32_t rowSize;
} STableComInfo;
typedef struct STableMeta {
int32_t vgId;
STableId id;
int8_t tableType;
char sTableName[TSDB_TABLE_FNAME_LEN]; // super table name
uint64_t suid; // super table id
int16_t sversion;
int16_t tversion;
STableComInfo tableInfo;
SSchema schema[]; // if the table is TSDB_CHILD_TABLE, schema is acquired by super table meta info
} STableMeta;
typedef struct STableMetaInfo {
STableMeta *pTableMeta; // table meta, cached in client side and acquired by name
uint32_t tableMetaSize;
size_t tableMetaCapacity;
SVgroupsInfo *vgroupList;
SArray *pVgroupTables; // SArray<SVgroupTableInfo>
/*
* 1. keep the vgroup index during the multi-vnode super table projection query
* 2. keep the vgroup index for multi-vnode insertion
*/
int32_t vgroupIndex;
SName name;
char aliasName[TSDB_TABLE_NAME_LEN]; // alias name of table specified in query sql
SArray *tagColList; // SArray<SColumn*>, involved tag columns
} STableMetaInfo;
struct SQInfo; // global merge operator
struct SQueryAttr; // query object
typedef struct STableFilter {
uint64_t uid;
SFilterInfo info;
} STableFilter;
typedef struct SQueryInfo {
int16_t command; // the command may be different for each subclause, so keep it seperately.
uint32_t type; // query/insert type
STimeWindow window; // the whole query time window
SInterval interval; // tumble time window
SSessionWindow sessionWindow; // session time window
SGroupbyExpr groupbyExpr; // groupby tags info
SArray * colList; // SArray<SColumn*>
SFieldInfo fieldsInfo;
SArray * exprList; // SArray<SExprInfo*>
SArray * exprList1; // final exprlist in case of arithmetic expression exists
SLimitVal limit;
SLimitVal slimit;
STagCond tagCond;
SArray * colCond;
SOrderVal order;
int16_t numOfTables;
int16_t curTableIdx;
STableMetaInfo **pTableMetaInfo;
struct STSBuf *tsBuf;
int16_t fillType; // final result fill type
int64_t * fillVal; // default value for fill
int32_t numOfFillVal; // fill value size
char * msg; // pointer to the pCmd->payload to keep error message temporarily
int64_t clauseLimit; // limit for current sub clause
int64_t prjOffset; // offset value in the original sql expression, only applied at client side
int64_t vgroupLimit; // table limit in case of super table projection query + global order + limit
int32_t udColumnId; // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX
bool distinct; // distinct tag or not
bool onlyHasTagCond;
int32_t round; // 0/1/....
int32_t bufLen;
char* buf;
bool udfCopy;
SArray *pUdfInfo;
struct SQInfo *pQInfo; // global merge operator
struct SQueryAttr *pQueryAttr; // query object
struct SQueryInfo *sibling; // sibling
SArray *pUpstream; // SArray<struct SQueryInfo>
struct SQueryInfo *pDownstream;
int32_t havingFieldNum;
bool stableQuery;
bool groupbyColumn;
bool simpleAgg;
bool arithmeticOnAgg;
bool projectionQuery;
bool hasFilter;
bool onlyTagQuery;
bool orderProjectQuery;
bool stateWindow;
bool globalMerge;
bool multigroupResult;
} SQueryInfo;
/**
* get the number of tags of this table
* @param pTableMeta
* @return
*/
int32_t tscGetNumOfTags(const STableMeta* pTableMeta);
/**
* get the number of columns of this table
* @param pTableMeta
* @return
*/
int32_t tscGetNumOfColumns(const STableMeta* pTableMeta);
/**
* get the basic info of this table
* @param pTableMeta
* @return
*/
STableComInfo tscGetTableInfo(const STableMeta* pTableMeta);
/**
* get the schema
* @param pTableMeta
* @return
*/
SSchema* tscGetTableSchema(const STableMeta* pTableMeta);
/**
* get the tag schema
* @param pMeta
* @return
*/
SSchema *tscGetTableTagSchema(const STableMeta *pMeta);
/**
* get the column schema according to the column index
* @param pMeta
* @param colIndex
* @return
*/
SSchema *tscGetTableColumnSchema(const STableMeta *pMeta, int32_t colIndex);
/**
* get the column schema according to the column id
* @param pTableMeta
* @param colId
* @return
*/
SSchema* tscGetColumnSchemaById(STableMeta* pTableMeta, int16_t colId);
/**
* create the table meta from the msg
* @param pTableMetaMsg
* @param size size of the table meta
* @return
*/
STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg);
#endif // TDENGINE_QTABLEUTIL_H

147
2.0/src/query/inc/qTsbuf.h
View File

@ -1,147 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_STSBUF_H
#define TDENGINE_STSBUF_H
#ifdef __cplusplus
extern "C" {
#endif
#include "os.h"
#include "taosdef.h"
#include "tvariant.h"
#define MEM_BUF_SIZE (1 << 20)
#define TS_COMP_FILE_MAGIC 0x87F5EC4C
#define TS_COMP_FILE_GROUP_MAX 512
typedef struct STSList {
char* rawBuf;
int32_t allocSize;
int32_t threshold;
int32_t len;
} STSList;
typedef struct STSElem {
TSKEY ts;
tVariant* tag;
int32_t id;
} STSElem;
typedef struct STSCursor {
int32_t vgroupIndex;
int32_t blockIndex;
int32_t tsIndex;
uint32_t order;
} STSCursor;
typedef struct STSBlock {
tVariant tag; // tag value
int32_t numOfElem; // number of elements
int32_t compLen; // size after compressed
int32_t padding; // 0xFFFFFFFF by default, after the payload
char* payload; // actual data that is compressed
} STSBlock;
/*
* The size of buffer file should not be greater than 2G,
* and the offset of int32_t type is enough
*/
typedef struct STSGroupBlockInfo {
int32_t id; // group id
int32_t offset; // offset set value in file
int32_t numOfBlocks; // number of total blocks
int32_t compLen; // compressed size
} STSGroupBlockInfo;
typedef struct STSGroupBlockInfoEx {
STSGroupBlockInfo info;
int32_t len; // length before compress
} STSGroupBlockInfoEx;
typedef struct STSBuf {
FILE* f;
char path[PATH_MAX];
uint32_t fileSize;
// todo use array
STSGroupBlockInfoEx* pData;
uint32_t numOfAlloc;
uint32_t numOfGroups;
char* assistBuf;
int32_t bufSize;
STSBlock block;
STSList tsData; // uncompressed raw ts data
uint64_t numOfTotal;
bool autoDelete;
bool remainOpen;
int32_t tsOrder; // order of timestamp in ts comp buffer
STSCursor cur;
} STSBuf;
typedef struct STSBufFileHeader {
uint32_t magic; // file magic number
uint32_t numOfGroup; // number of group stored in current file
int32_t tsOrder; // timestamp order in current file
} STSBufFileHeader;
STSBuf* tsBufCreate(bool autoDelete, int32_t order);
STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete);
STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t tsOrder, int32_t id);
void* tsBufDestroy(STSBuf* pTSBuf);
void tsBufAppend(STSBuf* pTSBuf, int32_t id, tVariant* tag, const char* pData, int32_t len);
int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf);
STSBuf* tsBufClone(STSBuf* pTSBuf);
STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id);
void tsBufFlush(STSBuf* pTSBuf);
void tsBufResetPos(STSBuf* pTSBuf);
bool tsBufNextPos(STSBuf* pTSBuf);
STSElem tsBufGetElem(STSBuf* pTSBuf);
STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t id, tVariant* tag);
STSCursor tsBufGetCursor(STSBuf* pTSBuf);
void tsBufSetTraverseOrder(STSBuf* pTSBuf, int32_t order);
void tsBufSetCursor(STSBuf* pTSBuf, STSCursor* pCur);
/**
* display all data in comp block file, for debug purpose only
* @param pTSBuf
*/
void tsBufDisplay(STSBuf* pTSBuf);
int32_t tsBufGetNumOfGroup(STSBuf* pTSBuf);
void tsBufGetGroupIdList(STSBuf* pTSBuf, int32_t* num, int32_t** id);
int32_t dumpFileBlockByGroupId(STSBuf* pTSBuf, int32_t id, void* buf, int32_t* len, int32_t* numOfBlocks);
STSElem tsBufFindElemStartPosByTag(STSBuf* pTSBuf, tVariant* pTag);
bool tsBufIsValidElem(STSElem* pElem);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_STSBUF_H

74
2.0/src/query/inc/qUdf.h
View File

@ -1,74 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_QUDF_H
#define TDENGINE_QUDF_H
enum { TSDB_UDF_FUNC_NORMAL = 0, TSDB_UDF_FUNC_INIT, TSDB_UDF_FUNC_FINALIZE, TSDB_UDF_FUNC_MERGE, TSDB_UDF_FUNC_DESTROY, TSDB_UDF_FUNC_MAX_NUM };
typedef struct SUdfInit{
int32_t maybe_null; /* 1 if function can return NULL */
uint32_t decimals; /* for real functions */
uint64_t length; /* For string functions */
char *ptr; /* free pointer for function data */
int32_t const_item; /* 0 if result is independent of arguments */
// script like lua/javascript
void* script_ctx;
void (*destroyCtxFunc)(void *script_ctx);
} SUdfInit;
typedef struct SUdfInfo {
int32_t functionId; // system assigned function id
int32_t funcType; // scalar function or aggregate function
int8_t resType; // result type
int16_t resBytes; // result byte
int32_t contLen; // content length
int32_t bufSize; //interbuf size
char *name; // function name
void *handle; // handle loaded in mem
void *funcs[TSDB_UDF_FUNC_MAX_NUM]; // function ptr
// for script like lua/javascript only
int isScript;
void *pScriptCtx;
SUdfInit init;
char *content;
char *path;
} SUdfInfo;
//script
typedef int32_t (*scriptInitFunc)(void *pCtx);
typedef void (*scriptNormalFunc)(void *pCtx, char* data, int16_t iType, int16_t iBytes, int32_t numOfRows,
int64_t* ptList, int64_t key, char* dataOutput, char* tsOutput, int32_t* numOfOutput, int16_t oType, int16_t oBytes);
typedef void (*scriptFinalizeFunc)(void *pCtx, int64_t key, char* dataOutput, int32_t* numOfOutput);
typedef void (*scriptMergeFunc)(void *pCtx, char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput);
typedef void (*scriptDestroyFunc)(void* pCtx);
// dynamic lib
typedef void (*udfNormalFunc)(char* data, int16_t itype, int16_t iBytes, int32_t numOfRows, int64_t* ts, char* dataOutput, char* interBuf,
char* tsOutput, int32_t* numOfOutput, int16_t oType, int16_t oBytes, SUdfInit* buf);
typedef int32_t (*udfInitFunc)(SUdfInit* data);
typedef void (*udfFinalizeFunc)(char* dataOutput, char *interBuf, int32_t* numOfOutput, SUdfInit* buf);
typedef void (*udfMergeFunc)(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf);
typedef void (*udfDestroyFunc)(SUdfInit* buf);
#endif // TDENGINE_QUDF_H

39
2.0/src/query/inc/queryLog.h
View File

@ -1,39 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_QUERY_LOG_H
#define TDENGINE_QUERY_LOG_H
#ifdef __cplusplus
extern "C" {
#endif
#include "tlog.h"
extern uint32_t qDebugFlag;
#define qFatal(...) do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", qDebugFlag, __VA_ARGS__); }} while(0)
#define qError(...) do { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", qDebugFlag, __VA_ARGS__); }} while(0)
#define qWarn(...) do { if (qDebugFlag & DEBUG_WARN) { taosPrintLog("QRY WARN ", qDebugFlag, __VA_ARGS__); }} while(0)
#define qInfo(...) do { if (qDebugFlag & DEBUG_INFO) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0)
#define qDebug(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0)
#define qTrace(...) do { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0)
#define qDump(a, l) do { if (qDebugFlag & DEBUG_DUMP) { taosDumpData((unsigned char *)a, l); }} while(0)
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_QUERY_LOG_H

931
2.0/src/query/inc/sql.y
View File

@ -1,931 +0,0 @@
//lemon parser file to generate sql parse by using finite-state-machine code used to parse sql
//usage: lemon sql.y
%token_prefix TK_
%token_type {SStrToken}
%default_type {SStrToken}
%extra_argument {SSqlInfo* pInfo}
%fallback ID BOOL TINYINT SMALLINT INTEGER BIGINT FLOAT DOUBLE STRING TIMESTAMP BINARY NCHAR.
%left OR.
%left AND.
%right NOT.
%left EQ NE ISNULL NOTNULL IS LIKE MATCH NMATCH GLOB BETWEEN IN.
%left GT GE LT LE.
%left BITAND BITOR LSHIFT RSHIFT.
%left PLUS MINUS.
%left DIVIDE TIMES.
%left STAR SLASH REM.
%left CONCAT.
%right UMINUS UPLUS BITNOT.
%include {
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include <stdbool.h>
#include "qSqlparser.h"
#include "tcmdtype.h"
#include "ttoken.h"
#include "ttokendef.h"
#include "tutil.h"
#include "tvariant.h"
#include "parserInt.h"
}
%syntax_error {
pInfo->valid = false;
int32_t outputBufLen = tListLen(pInfo->msg);
int32_t len = 0;
if(TOKEN.z) {
char msg[] = "syntax error near \"%s\"";
int32_t sqlLen = strlen(&TOKEN.z[0]);
if (sqlLen + sizeof(msg)/sizeof(msg[0]) + 1 > outputBufLen) {
char tmpstr[128] = {0};
memcpy(tmpstr, &TOKEN.z[0], sizeof(tmpstr)/sizeof(tmpstr[0]) - 1);
len = sprintf(pInfo->msg, msg, tmpstr);
} else {
len = sprintf(pInfo->msg, msg, &TOKEN.z[0]);
}
} else {
len = sprintf(pInfo->msg, "Incomplete SQL statement");
}
assert(len <= outputBufLen);
}
%parse_accept {}
program ::= cmd. {}
//////////////////////////////////THE SHOW STATEMENT///////////////////////////////////////////
cmd ::= SHOW DATABASES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_DB, 0, 0);}
cmd ::= SHOW TOPICS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_TP, 0, 0);}
cmd ::= SHOW FUNCTIONS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_FUNCTION, 0, 0);}
cmd ::= SHOW MNODES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_MNODE, 0, 0);}
cmd ::= SHOW DNODES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_DNODE, 0, 0);}
cmd ::= SHOW ACCOUNTS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_ACCT, 0, 0);}
cmd ::= SHOW USERS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_USER, 0, 0);}
cmd ::= SHOW MODULES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_MODULE, 0, 0); }
cmd ::= SHOW QUERIES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_QUERIES, 0, 0); }
cmd ::= SHOW CONNECTIONS.{ setShowOptions(pInfo, TSDB_MGMT_TABLE_CONNS, 0, 0);}
cmd ::= SHOW STREAMS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_STREAMS, 0, 0); }
cmd ::= SHOW VARIABLES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_VARIABLES, 0, 0); }
cmd ::= SHOW SCORES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_SCORES, 0, 0); }
cmd ::= SHOW GRANTS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_GRANTS, 0, 0); }
cmd ::= SHOW VNODES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_VNODES, 0, 0); }
cmd ::= SHOW VNODES ids(X). { setShowOptions(pInfo, TSDB_MGMT_TABLE_VNODES, &X, 0); }
%type dbPrefix {SStrToken}
dbPrefix(A) ::=. {A.n = 0; A.type = 0;}
dbPrefix(A) ::= ids(X) DOT. {A = X; }
%type cpxName {SStrToken}
cpxName(A) ::= . {A.n = 0; }
cpxName(A) ::= DOT ids(Y). {A = Y; A.n += 1; }
cmd ::= SHOW CREATE TABLE ids(X) cpxName(Y). {
X.n += Y.n;
setDCLSqlElems(pInfo, TSDB_SQL_SHOW_CREATE_TABLE, 1, &X);
}
cmd ::= SHOW CREATE STABLE ids(X) cpxName(Y). {
X.n += Y.n;
setDCLSqlElems(pInfo, TSDB_SQL_SHOW_CREATE_STABLE, 1, &X);
}
cmd ::= SHOW CREATE DATABASE ids(X). {
setDCLSqlElems(pInfo, TSDB_SQL_SHOW_CREATE_DATABASE, 1, &X);
}
cmd ::= SHOW dbPrefix(X) TABLES. {
setShowOptions(pInfo, TSDB_MGMT_TABLE_TABLE, &X, 0);
}
cmd ::= SHOW dbPrefix(X) TABLES LIKE ids(Y). {
setShowOptions(pInfo, TSDB_MGMT_TABLE_TABLE, &X, &Y);
}
cmd ::= SHOW dbPrefix(X) STABLES. {
setShowOptions(pInfo, TSDB_MGMT_TABLE_METRIC, &X, 0);
}
cmd ::= SHOW dbPrefix(X) STABLES LIKE ids(Y). {
SStrToken token;
tSetDbName(&token, &X);
setShowOptions(pInfo, TSDB_MGMT_TABLE_METRIC, &token, &Y);
}
cmd ::= SHOW dbPrefix(X) VGROUPS. {
SStrToken token;
tSetDbName(&token, &X);
setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, 0);
}
cmd ::= SHOW dbPrefix(X) VGROUPS ids(Y). {
SStrToken token;
tSetDbName(&token, &X);
setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, &Y);
}
//drop configure for tables
cmd ::= DROP TABLE ifexists(Y) ids(X) cpxName(Z). {
X.n += Z.n;
setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &X, &Y, -1, -1);
}
//drop stable
cmd ::= DROP STABLE ifexists(Y) ids(X) cpxName(Z). {
X.n += Z.n;
setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &X, &Y, -1, TSDB_SUPER_TABLE);
}
cmd ::= DROP DATABASE ifexists(Y) ids(X). { setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &X, &Y, TSDB_DB_TYPE_DEFAULT, -1); }
cmd ::= DROP TOPIC ifexists(Y) ids(X). { setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &X, &Y, TSDB_DB_TYPE_TOPIC, -1); }
cmd ::= DROP FUNCTION ids(X). { setDropFuncInfo(pInfo, TSDB_SQL_DROP_FUNCTION, &X); }
cmd ::= DROP DNODE ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_DROP_DNODE, 1, &X); }
cmd ::= DROP USER ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_DROP_USER, 1, &X); }
cmd ::= DROP ACCOUNT ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_DROP_ACCT, 1, &X); }
/////////////////////////////////THE USE STATEMENT//////////////////////////////////////////
cmd ::= USE ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_USE_DB, 1, &X);}
/////////////////////////////////THE DESCRIBE STATEMENT/////////////////////////////////////
cmd ::= DESCRIBE ids(X) cpxName(Y). {
X.n += Y.n;
setDCLSqlElems(pInfo, TSDB_SQL_DESCRIBE_TABLE, 1, &X);
}
cmd ::= DESC ids(X) cpxName(Y). {
X.n += Y.n;
setDCLSqlElems(pInfo, TSDB_SQL_DESCRIBE_TABLE, 1, &X);
}
/////////////////////////////////THE ALTER STATEMENT////////////////////////////////////////
cmd ::= ALTER USER ids(X) PASS ids(Y). { setAlterUserSql(pInfo, TSDB_ALTER_USER_PASSWD, &X, &Y, NULL); }
cmd ::= ALTER USER ids(X) PRIVILEGE ids(Y). { setAlterUserSql(pInfo, TSDB_ALTER_USER_PRIVILEGES, &X, NULL, &Y);}
cmd ::= ALTER DNODE ids(X) ids(Y). { setDCLSqlElems(pInfo, TSDB_SQL_CFG_DNODE, 2, &X, &Y); }
cmd ::= ALTER DNODE ids(X) ids(Y) ids(Z). { setDCLSqlElems(pInfo, TSDB_SQL_CFG_DNODE, 3, &X, &Y, &Z); }
cmd ::= ALTER LOCAL ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_CFG_LOCAL, 1, &X); }
cmd ::= ALTER LOCAL ids(X) ids(Y). { setDCLSqlElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &X, &Y); }
cmd ::= ALTER DATABASE ids(X) alter_db_optr(Y). { SStrToken t = {0}; setCreateDbInfo(pInfo, TSDB_SQL_ALTER_DB, &X, &Y, &t);}
cmd ::= ALTER TOPIC ids(X) alter_topic_optr(Y). { SStrToken t = {0}; setCreateDbInfo(pInfo, TSDB_SQL_ALTER_DB, &X, &Y, &t);}
cmd ::= ALTER ACCOUNT ids(X) acct_optr(Z). { setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &X, NULL, &Z);}
cmd ::= ALTER ACCOUNT ids(X) PASS ids(Y) acct_optr(Z). { setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &X, &Y, &Z);}
////////////////////////////// COMPACT STATEMENT //////////////////////////////////////////////
cmd ::= COMPACT VNODES IN LP exprlist(Y) RP. { setCompactVnodeSql(pInfo, TSDB_SQL_COMPACT_VNODE, Y);}
// An IDENTIFIER can be a generic identifier, or one of several keywords.
// Any non-standard keyword can also be an identifier.
// And "ids" is an identifer-or-string.
%type ids {SStrToken}
ids(A) ::= ID(X). {A = X; }
ids(A) ::= STRING(X). {A = X; }
%type ifexists {SStrToken}
ifexists(X) ::= IF EXISTS. { X.n = 1;}
ifexists(X) ::= . { X.n = 0;}
%type ifnotexists {SStrToken}
ifnotexists(X) ::= IF NOT EXISTS. { X.n = 1;}
ifnotexists(X) ::= . { X.n = 0;}
/////////////////////////////////THE CREATE STATEMENT///////////////////////////////////////
//create option for dnode/db/user/account
cmd ::= CREATE DNODE ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_CREATE_DNODE, 1, &X);}
cmd ::= CREATE ACCOUNT ids(X) PASS ids(Y) acct_optr(Z).
{ setCreateAcctSql(pInfo, TSDB_SQL_CREATE_ACCT, &X, &Y, &Z);}
cmd ::= CREATE DATABASE ifnotexists(Z) ids(X) db_optr(Y). { setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &X, &Y, &Z);}
cmd ::= CREATE TOPIC ifnotexists(Z) ids(X) topic_optr(Y). { setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &X, &Y, &Z);}
cmd ::= CREATE FUNCTION ids(X) AS ids(Y) OUTPUTTYPE typename(Z) bufsize(B). { setCreateFuncInfo(pInfo, TSDB_SQL_CREATE_FUNCTION, &X, &Y, &Z, &B, 1);}
cmd ::= CREATE AGGREGATE FUNCTION ids(X) AS ids(Y) OUTPUTTYPE typename(Z) bufsize(B). { setCreateFuncInfo(pInfo, TSDB_SQL_CREATE_FUNCTION, &X, &Y, &Z, &B, 2);}
cmd ::= CREATE USER ids(X) PASS ids(Y). { setCreateUserSql(pInfo, &X, &Y);}
bufsize(Y) ::= . { Y.n = 0; }
bufsize(Y) ::= BUFSIZE INTEGER(X). { Y = X; }
pps(Y) ::= . { Y.n = 0; }
pps(Y) ::= PPS INTEGER(X). { Y = X; }
tseries(Y) ::= . { Y.n = 0; }
tseries(Y) ::= TSERIES INTEGER(X). { Y = X; }
dbs(Y) ::= . { Y.n = 0; }
dbs(Y) ::= DBS INTEGER(X). { Y = X; }
streams(Y) ::= . { Y.n = 0; }
streams(Y) ::= STREAMS INTEGER(X). { Y = X; }
storage(Y) ::= . { Y.n = 0; }
storage(Y) ::= STORAGE INTEGER(X). { Y = X; }
qtime(Y) ::= . { Y.n = 0; }
qtime(Y) ::= QTIME INTEGER(X). { Y = X; }
users(Y) ::= . { Y.n = 0; }
users(Y) ::= USERS INTEGER(X). { Y = X; }
conns(Y) ::= . { Y.n = 0; }
conns(Y) ::= CONNS INTEGER(X). { Y = X; }
state(Y) ::= . { Y.n = 0; }
state(Y) ::= STATE ids(X). { Y = X; }
%type acct_optr {SCreateAcctInfo}
acct_optr(Y) ::= pps(C) tseries(D) storage(P) streams(F) qtime(Q) dbs(E) users(K) conns(L) state(M). {
Y.maxUsers = (K.n>0)?atoi(K.z):-1;
Y.maxDbs = (E.n>0)?atoi(E.z):-1;
Y.maxTimeSeries = (D.n>0)?atoi(D.z):-1;
Y.maxStreams = (F.n>0)?atoi(F.z):-1;
Y.maxPointsPerSecond = (C.n>0)?atoi(C.z):-1;
Y.maxStorage = (P.n>0)?strtoll(P.z, NULL, 10):-1;
Y.maxQueryTime = (Q.n>0)?strtoll(Q.z, NULL, 10):-1;
Y.maxConnections = (L.n>0)?atoi(L.z):-1;
Y.stat = M;
}
%type intitemlist {SArray*}
%destructor intitemlist {taosArrayDestroy($$);}
%type intitem {tVariant}
intitemlist(A) ::= intitemlist(X) COMMA intitem(Y). { A = tVariantListAppend(X, &Y, -1); }
intitemlist(A) ::= intitem(X). { A = tVariantListAppend(NULL, &X, -1); }
intitem(A) ::= INTEGER(X). { toTSDBType(X.type); taosVariantCreate(&A, &X); }
%type keep {SArray*}
%destructor keep {taosArrayDestroy($$);}
keep(Y) ::= KEEP intitemlist(X). { Y = X; }
cache(Y) ::= CACHE INTEGER(X). { Y = X; }
replica(Y) ::= REPLICA INTEGER(X). { Y = X; }
quorum(Y) ::= QUORUM INTEGER(X). { Y = X; }
days(Y) ::= DAYS INTEGER(X). { Y = X; }
minrows(Y) ::= MINROWS INTEGER(X). { Y = X; }
maxrows(Y) ::= MAXROWS INTEGER(X). { Y = X; }
blocks(Y) ::= BLOCKS INTEGER(X). { Y = X; }
ctime(Y) ::= CTIME INTEGER(X). { Y = X; }
wal(Y) ::= WAL INTEGER(X). { Y = X; }
fsync(Y) ::= FSYNC INTEGER(X). { Y = X; }
comp(Y) ::= COMP INTEGER(X). { Y = X; }
prec(Y) ::= PRECISION STRING(X). { Y = X; }
update(Y) ::= UPDATE INTEGER(X). { Y = X; }
cachelast(Y) ::= CACHELAST INTEGER(X). { Y = X; }
partitions(Y) ::= PARTITIONS INTEGER(X). { Y = X; }
%type db_optr {SCreateDbInfo}
db_optr(Y) ::= . {setDefaultCreateDbOption(&Y); Y.dbType = TSDB_DB_TYPE_DEFAULT;}
db_optr(Y) ::= db_optr(Z) cache(X). { Y = Z; Y.cacheBlockSize = strtol(X.z, NULL, 10); }
db_optr(Y) ::= db_optr(Z) replica(X). { Y = Z; Y.replica = strtol(X.z, NULL, 10); }
db_optr(Y) ::= db_optr(Z) quorum(X). { Y = Z; Y.quorum = strtol(X.z, NULL, 10); }
db_optr(Y) ::= db_optr(Z) days(X). { Y = Z; Y.daysPerFile = strtol(X.z, NULL, 10); }
db_optr(Y) ::= db_optr(Z) minrows(X). { Y = Z; Y.minRowsPerBlock = strtod(X.z, NULL); }
db_optr(Y) ::= db_optr(Z) maxrows(X). { Y = Z; Y.maxRowsPerBlock = strtod(X.z, NULL); }
db_optr(Y) ::= db_optr(Z) blocks(X). { Y = Z; Y.numOfBlocks = strtol(X.z, NULL, 10); }
db_optr(Y) ::= db_optr(Z) ctime(X). { Y = Z; Y.commitTime = strtol(X.z, NULL, 10); }
db_optr(Y) ::= db_optr(Z) wal(X). { Y = Z; Y.walLevel = strtol(X.z, NULL, 10); }
db_optr(Y) ::= db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = strtol(X.z, NULL, 10); }
db_optr(Y) ::= db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); }
db_optr(Y) ::= db_optr(Z) prec(X). { Y = Z; Y.precision = X; }
db_optr(Y) ::= db_optr(Z) keep(X). { Y = Z; Y.keep = X; }
db_optr(Y) ::= db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); }
db_optr(Y) ::= db_optr(Z) cachelast(X). { Y = Z; Y.cachelast = strtol(X.z, NULL, 10); }
%type topic_optr {SCreateDbInfo}
//topic_optr(Y) ::= db_optr(Z). { Y = Z; Y.dbType = TSDB_DB_TYPE_TOPIC; }
//topic_optr(Y) ::= topic_optr(Z) partitions(X). { Y = Z; Y.partitions = strtol(X.z, NULL, 10); }
%type alter_db_optr {SCreateDbInfo}
alter_db_optr(Y) ::= . { setDefaultCreateDbOption(&Y); Y.dbType = TSDB_DB_TYPE_DEFAULT;}
alter_db_optr(Y) ::= alter_db_optr(Z) replica(X). { Y = Z; Y.replica = strtol(X.z, NULL, 10); }
alter_db_optr(Y) ::= alter_db_optr(Z) quorum(X). { Y = Z; Y.quorum = strtol(X.z, NULL, 10); }
alter_db_optr(Y) ::= alter_db_optr(Z) keep(X). { Y = Z; Y.keep = X; }
alter_db_optr(Y) ::= alter_db_optr(Z) blocks(X). { Y = Z; Y.numOfBlocks = strtol(X.z, NULL, 10); }
alter_db_optr(Y) ::= alter_db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); }
alter_db_optr(Y) ::= alter_db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); }
alter_db_optr(Y) ::= alter_db_optr(Z) cachelast(X). { Y = Z; Y.cachelast = strtol(X.z, NULL, 10); }
// dynamically update the following two parameters are not allowed.
//alter_db_optr(Y) ::= alter_db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = strtol(X.z, NULL, 10); }
//alter_db_optr(Y) ::= alter_db_optr(Z) wal(X). { Y = Z; Y.walLevel = strtol(X.z, NULL, 10); } not support yet
%type alter_topic_optr {SCreateDbInfo}
alter_topic_optr(Y) ::= alter_db_optr(Z). { Y = Z; Y.dbType = TSDB_DB_TYPE_TOPIC; }
alter_topic_optr(Y) ::= alter_topic_optr(Z) partitions(X). { Y = Z; Y.partitions = strtol(X.z, NULL, 10); }
%type typename {SField}
typename(A) ::= ids(X). {
X.type = 0;
tSetColumnType (&A, &X);
}
//define binary type, e.g., binary(10), nchar(10)
typename(A) ::= ids(X) LP signed(Y) RP. {
if (Y <= 0) {
X.type = 0;
tSetColumnType(&A, &X);
} else {
X.type = -Y; // negative value of name length
tSetColumnType(&A, &X);
}
}
// define the unsigned number type
typename(A) ::= ids(X) UNSIGNED(Z). {
X.type = 0;
X.n = ((Z.z + Z.n) - X.z);
tSetColumnType (&A, &X);
}
%type signed {int64_t}
signed(A) ::= INTEGER(X). { A = strtol(X.z, NULL, 10); }
signed(A) ::= PLUS INTEGER(X). { A = strtol(X.z, NULL, 10); }
signed(A) ::= MINUS INTEGER(X). { A = -strtol(X.z, NULL, 10);}
////////////////////////////////// The CREATE TABLE statement ///////////////////////////////
cmd ::= CREATE TABLE create_table_args. {}
cmd ::= CREATE TABLE create_stable_args. {}
cmd ::= CREATE STABLE create_stable_args. {}
cmd ::= CREATE TABLE create_table_list(Z). { pInfo->type = TSDB_SQL_CREATE_TABLE; pInfo->pCreateTableInfo = Z;}
%type create_table_list{SCreateTableSql*}
%destructor create_table_list{destroyCreateTableSql($$);}
create_table_list(A) ::= create_from_stable(Z). {
SCreateTableSql* pCreateTable = calloc(1, sizeof(SCreateTableSql));
pCreateTable->childTableInfo = taosArrayInit(4, sizeof(SCreatedTableInfo));
taosArrayPush(pCreateTable->childTableInfo, &Z);
pCreateTable->type = TSQL_CREATE_TABLE_FROM_STABLE;
A = pCreateTable;
}
create_table_list(A) ::= create_table_list(X) create_from_stable(Z). {
taosArrayPush(X->childTableInfo, &Z);
A = X;
}
%type create_table_args{SCreateTableSql*}
create_table_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) LP columnlist(X) RP. {
A = tSetCreateTableInfo(X, NULL, NULL, TSQL_CREATE_TABLE);
setSqlInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE);
V.n += Z.n;
setCreatedTableName(pInfo, &V, &U);
}
// create super table
%type create_stable_args{SCreateTableSql*}
create_stable_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) LP columnlist(X) RP TAGS LP columnlist(Y) RP. {
A = tSetCreateTableInfo(X, Y, NULL, TSQL_CREATE_STABLE);
setSqlInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE);
V.n += Z.n;
setCreatedTableName(pInfo, &V, &U);
}
// create table by using super table
// create table table_name using super_table_name tags(tag_values1, tag_values2)
%type create_from_stable{SCreatedTableInfo}
create_from_stable(A) ::= ifnotexists(U) ids(V) cpxName(Z) USING ids(X) cpxName(F) TAGS LP tagitemlist(Y) RP. {
X.n += F.n;
V.n += Z.n;
A = createNewChildTableInfo(&X, NULL, Y, &V, &U);
}
create_from_stable(A) ::= ifnotexists(U) ids(V) cpxName(Z) USING ids(X) cpxName(F) LP tagNamelist(P) RP TAGS LP tagitemlist(Y) RP. {
X.n += F.n;
V.n += Z.n;
A = createNewChildTableInfo(&X, P, Y, &V, &U);
}
%type tagNamelist{SArray*}
%destructor tagNamelist {taosArrayDestroy($$);}
tagNamelist(A) ::= tagNamelist(X) COMMA ids(Y). {taosArrayPush(X, &Y); A = X; }
tagNamelist(A) ::= ids(X). {A = taosArrayInit(4, sizeof(SStrToken)); taosArrayPush(A, &X);}
// create stream
// create table table_name as select count(*) from super_table_name interval(time)
create_table_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) AS select(S). {
A = tSetCreateTableInfo(NULL, NULL, S, TSQL_CREATE_STREAM);
setSqlInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE);
V.n += Z.n;
setCreatedTableName(pInfo, &V, &U);
}
%type column{SField}
%type columnlist{SArray*}
%destructor columnlist {taosArrayDestroy($$);}
columnlist(A) ::= columnlist(X) COMMA column(Y). {taosArrayPush(X, &Y); A = X; }
columnlist(A) ::= column(X). {A = taosArrayInit(4, sizeof(SField)); taosArrayPush(A, &X);}
// The information used for a column is the name and type of column:
// tinyint smallint int bigint float double bool timestamp binary(x) nchar(x)
column(A) ::= ids(X) typename(Y). {
tSetColumnInfo(&A, &X, &Y);
}
%type tagitemlist1 {SArray*}
%destructor tagitemlist1 {taosArrayDestroy($$);}
%type tagitem1 {SToken}
%type tagitemlist {SArray*}
%destructor tagitemlist {taosArrayDestroy($$);}
%type tagitem {tVariant}
tagitemlist(A) ::= tagitemlist(X) COMMA tagitem(Y). { A = tVariantListAppend(X, &Y, -1); }
tagitemlist(A) ::= tagitem(X). { A = tVariantListAppend(NULL, &X, -1); }
tagitem(A) ::= INTEGER(X). { toTSDBType(X.type); taosVariantCreate(&A, &X); }
tagitem(A) ::= FLOAT(X). { toTSDBType(X.type); taosVariantCreate(&A, &X); }
tagitem(A) ::= STRING(X). { toTSDBType(X.type); taosVariantCreate(&A, &X); }
tagitem(A) ::= BOOL(X). { toTSDBType(X.type); taosVariantCreate(&A, &X); }
tagitem(A) ::= NULL(X). { X.type = 0; taosVariantCreate(&A, &X); }
tagitem(A) ::= NOW(X). { X.type = TSDB_DATA_TYPE_TIMESTAMP; taosVariantCreate(&A, &X);}
tagitem(A) ::= MINUS(X) INTEGER(Y).{
X.n += Y.n;
X.type = Y.type;
toTSDBType(X.type);
taosVariantCreate(&A, &X);
}
tagitem(A) ::= MINUS(X) FLOAT(Y). {
X.n += Y.n;
X.type = Y.type;
toTSDBType(X.type);
taosVariantCreate(&A, &X);
}
tagitem(A) ::= PLUS(X) INTEGER(Y). {
X.n += Y.n;
X.type = Y.type;
toTSDBType(X.type);
taosVariantCreate(&A, &X);
}
tagitem(A) ::= PLUS(X) FLOAT(Y). {
X.n += Y.n;
X.type = Y.type;
toTSDBType(X.type);
taosVariantCreate(&A, &X);
}
//////////////////////// The SELECT statement /////////////////////////////////
%type select {SSqlNode*}
%destructor select {destroySqlNode($$);}
select(A) ::= SELECT(T) selcollist(W) from(X) where_opt(Y) interval_option(K) sliding_opt(S) session_option(H) windowstate_option(D) fill_opt(F)groupby_opt(P) having_opt(N) orderby_opt(Z) slimit_opt(G) limit_opt(L). {
A = tSetQuerySqlNode(&T, W, X, Y, P, Z, &K, &H, &D, &S, F, &L, &G, N);
}
select(A) ::= LP select(B) RP. {A = B;}
%type union {SArray*}
%destructor union {destroyAllSqlNode($$);}
union(Y) ::= select(X). { Y = setSubclause(NULL, X); }
union(Y) ::= union(Z) UNION ALL select(X). { Y = appendSelectClause(Z, X); }
cmd ::= union(X). { setSqlInfo(pInfo, X, NULL, TSDB_SQL_SELECT); }
// Support for the SQL exprssion without from & where subclauses, e.g.,
// select database()
// select server_version()
// select client_version()
// select server_state()
select(A) ::= SELECT(T) selcollist(W). {
A = tSetQuerySqlNode(&T, W, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
}
// selcollist is a list of expressions that are to become the return
// values of the SELECT statement. The "*" in statements like
// "SELECT * FROM ..." is encoded as a special expression with an opcode of TK_ALL.
%type selcollist {SArray*}
%destructor selcollist {tSqlExprListDestroy($$);}
%type sclp {SArray*}
%destructor sclp {tSqlExprListDestroy($$);}
sclp(A) ::= selcollist(X) COMMA. {A = X;}
sclp(A) ::= . {A = 0;}
selcollist(A) ::= sclp(P) distinct(Z) expr(X) as(Y). {
A = tSqlExprListAppend(P, X, Z.n? &Z:0, Y.n?&Y:0);
}
selcollist(A) ::= sclp(P) STAR. {
tSqlExpr *pNode = tSqlExprCreateIdValue(NULL, TK_ALL);
A = tSqlExprListAppend(P, pNode, 0, 0);
}
// An option "AS <id>" phrase that can follow one of the expressions that
// define the result set, or one of the tables in the FROM clause.
%type as {SStrToken}
as(X) ::= AS ids(Y). { X = Y; }
as(X) ::= ids(Y). { X = Y; }
as(X) ::= . { X.n = 0; }
%type distinct {SStrToken}
distinct(X) ::= DISTINCT(Y). { X = Y; }
distinct(X) ::= . { X.n = 0;}
// A complete FROM clause.
%type from {SRelationInfo*}
%destructor from {destroyRelationInfo($$);}
from(A) ::= FROM tablelist(X). {A = X;}
from(A) ::= FROM sub(X). {A = X;}
%type sub {SRelationInfo*}
%destructor sub {destroyRelationInfo($$);}
sub(A) ::= LP union(Y) RP. {A = addSubqueryElem(NULL, Y, NULL);}
sub(A) ::= LP union(Y) RP ids(Z). {A = addSubqueryElem(NULL, Y, &Z);}
sub(A) ::= sub(X) COMMA LP union(Y) RP ids(Z).{A = addSubqueryElem(X, Y, &Z);}
%type tablelist {SRelationInfo*}
%destructor tablelist {destroyRelationInfo($$);}
tablelist(A) ::= ids(X) cpxName(Y). {
X.n += Y.n;
A = setTableNameList(NULL, &X, NULL);
}
tablelist(A) ::= ids(X) cpxName(Y) ids(Z). {
X.n += Y.n;
A = setTableNameList(NULL, &X, &Z);
}
tablelist(A) ::= tablelist(Y) COMMA ids(X) cpxName(Z). {
X.n += Z.n;
A = setTableNameList(Y, &X, NULL);
}
tablelist(A) ::= tablelist(Y) COMMA ids(X) cpxName(Z) ids(F). {
X.n += Z.n;
A = setTableNameList(Y, &X, &F);
}
// The value of interval should be the form of "number+[a,s,m,h,d,n,y]" or "now"
%type tmvar {SStrToken}
tmvar(A) ::= VARIABLE(X). {A = X;}
%type interval_option {SIntervalVal}
interval_option(N) ::= intervalKey(A) LP tmvar(E) RP. {N.interval = E; N.offset.n = 0; N.token = A;}
interval_option(N) ::= intervalKey(A) LP tmvar(E) COMMA tmvar(X) RP. {N.interval = E; N.offset = X; N.token = A;}
interval_option(N) ::= . {memset(&N, 0, sizeof(N));}
%type intervalKey {int32_t}
intervalKey(A) ::= INTERVAL. {A = TK_INTERVAL;}
intervalKey(A) ::= EVERY. {A = TK_EVERY; }
%type session_option {SSessionWindowVal}
session_option(X) ::= . {X.col.n = 0; X.gap.n = 0;}
session_option(X) ::= SESSION LP ids(V) cpxName(Z) COMMA tmvar(Y) RP. {
V.n += Z.n;
X.col = V;
X.gap = Y;
}
%type windowstate_option {SWindowStateVal}
windowstate_option(X) ::= . { X.col.n = 0; X.col.z = NULL;}
windowstate_option(X) ::= STATE_WINDOW LP ids(V) RP. { X.col = V; }
%type fill_opt {SArray*}
%destructor fill_opt {taosArrayDestroy($$);}
fill_opt(N) ::= . { N = 0; }
fill_opt(N) ::= FILL LP ID(Y) COMMA tagitemlist(X) RP. {
tVariant A = {0};
toTSDBType(Y.type);
taosVariantCreate(&A, &Y);
tVariantListInsert(X, &A, -1, 0);
N = X;
}
fill_opt(N) ::= FILL LP ID(Y) RP. {
toTSDBType(Y.type);
N = tVariantListAppendToken(NULL, &Y, -1);
}
%type sliding_opt {SStrToken}
sliding_opt(K) ::= SLIDING LP tmvar(E) RP. {K = E; }
sliding_opt(K) ::= . {K.n = 0; K.z = NULL; K.type = 0; }
%type orderby_opt {SArray*}
%destructor orderby_opt {taosArrayDestroy($$);}
%type sortlist {SArray*}
%destructor sortlist {taosArrayDestroy($$);}
%type sortitem {tVariant}
%destructor sortitem {tVariantDestroy(&$$);}
orderby_opt(A) ::= . {A = 0;}
orderby_opt(A) ::= ORDER BY sortlist(X). {A = X;}
sortlist(A) ::= sortlist(X) COMMA item(Y) sortorder(Z). {
A = tVariantListAppend(X, &Y, Z);
}
sortlist(A) ::= item(Y) sortorder(Z). {
A = tVariantListAppend(NULL, &Y, Z);
}
%type item {tVariant}
item(A) ::= ids(X) cpxName(Y). {
toTSDBType(X.type);
X.n += Y.n;
taosVariantCreate(&A, &X);
}
%type sortorder {int}
sortorder(A) ::= ASC. { A = TSDB_ORDER_ASC; }
sortorder(A) ::= DESC. { A = TSDB_ORDER_DESC;}
sortorder(A) ::= . { A = TSDB_ORDER_ASC; } // Ascending order by default
//group by clause
%type groupby_opt {SArray*}
%destructor groupby_opt {taosArrayDestroy($$);}
%type grouplist {SArray*}
%destructor grouplist {taosArrayDestroy($$);}
groupby_opt(A) ::= . { A = 0;}
groupby_opt(A) ::= GROUP BY grouplist(X). { A = X;}
grouplist(A) ::= grouplist(X) COMMA item(Y). {
A = tVariantListAppend(X, &Y, -1);
}
grouplist(A) ::= item(X). {
A = tVariantListAppend(NULL, &X, -1);
}
//having clause, ignore the input condition in having
%type having_opt {tSqlExpr*}
%destructor having_opt {tSqlExprDestroy($$);}
having_opt(A) ::=. {A = 0;}
having_opt(A) ::= HAVING expr(X). {A = X;}
//limit-offset subclause
%type limit_opt {SLimitVal}
limit_opt(A) ::= . {A.limit = -1; A.offset = 0;}
limit_opt(A) ::= LIMIT signed(X). {A.limit = X; A.offset = 0;}
limit_opt(A) ::= LIMIT signed(X) OFFSET signed(Y).
{ A.limit = X; A.offset = Y;}
limit_opt(A) ::= LIMIT signed(X) COMMA signed(Y).
{ A.limit = Y; A.offset = X;}
%type slimit_opt {SLimitVal}
slimit_opt(A) ::= . {A.limit = -1; A.offset = 0;}
slimit_opt(A) ::= SLIMIT signed(X). {A.limit = X; A.offset = 0;}
slimit_opt(A) ::= SLIMIT signed(X) SOFFSET signed(Y).
{A.limit = X; A.offset = Y;}
slimit_opt(A) ::= SLIMIT signed(X) COMMA signed(Y).
{A.limit = Y; A.offset = X;}
%type where_opt {tSqlExpr*}
%destructor where_opt {tSqlExprDestroy($$);}
where_opt(A) ::= . {A = 0;}
where_opt(A) ::= WHERE expr(X). {A = X;}
/////////////////////////// Expression Processing /////////////////////////////
//
%type expr {tSqlExpr*}
%destructor expr {tSqlExprDestroy($$);}
expr(A) ::= LP(X) expr(Y) RP(Z). {A = Y; A->exprToken.z = X.z; A->exprToken.n = (Z.z - X.z + 1);}
expr(A) ::= ID(X). { A = tSqlExprCreateIdValue(&X, TK_ID);}
expr(A) ::= ID(X) DOT ID(Y). { X.n += (1+Y.n); A = tSqlExprCreateIdValue(&X, TK_ID);}
expr(A) ::= ID(X) DOT STAR(Y). { X.n += (1+Y.n); A = tSqlExprCreateIdValue(&X, TK_ALL);}
expr(A) ::= INTEGER(X). { A = tSqlExprCreateIdValue(&X, TK_INTEGER);}
expr(A) ::= MINUS(X) INTEGER(Y). { X.n += Y.n; X.type = TK_INTEGER; A = tSqlExprCreateIdValue(&X, TK_INTEGER);}
expr(A) ::= PLUS(X) INTEGER(Y). { X.n += Y.n; X.type = TK_INTEGER; A = tSqlExprCreateIdValue(&X, TK_INTEGER);}
expr(A) ::= FLOAT(X). { A = tSqlExprCreateIdValue(&X, TK_FLOAT);}
expr(A) ::= MINUS(X) FLOAT(Y). { X.n += Y.n; X.type = TK_FLOAT; A = tSqlExprCreateIdValue(&X, TK_FLOAT);}
expr(A) ::= PLUS(X) FLOAT(Y). { X.n += Y.n; X.type = TK_FLOAT; A = tSqlExprCreateIdValue(&X, TK_FLOAT);}
expr(A) ::= STRING(X). { A = tSqlExprCreateIdValue(&X, TK_STRING);}
expr(A) ::= NOW(X). { A = tSqlExprCreateIdValue(&X, TK_NOW); }
expr(A) ::= VARIABLE(X). { A = tSqlExprCreateIdValue(&X, TK_VARIABLE);}
expr(A) ::= PLUS(X) VARIABLE(Y). { X.n += Y.n; X.type = TK_VARIABLE; A = tSqlExprCreateIdValue(&X, TK_VARIABLE);}
expr(A) ::= MINUS(X) VARIABLE(Y). { X.n += Y.n; X.type = TK_VARIABLE; A = tSqlExprCreateIdValue(&X, TK_VARIABLE);}
expr(A) ::= BOOL(X). { A = tSqlExprCreateIdValue(&X, TK_BOOL);}
expr(A) ::= NULL(X). { A = tSqlExprCreateIdValue(&X, TK_NULL);}
// ordinary functions: min(x), max(x), top(k, 20)
expr(A) ::= ID(X) LP exprlist(Y) RP(E). { (pInfo->funcs, &X); A = tSqlExprCreateFunction(Y, &X, &E, X.type); }
// for parsing sql functions with wildcard for parameters. e.g., count(*)/first(*)/last(*) operation
expr(A) ::= ID(X) LP STAR RP(Y). { tStrTokenAppend(pInfo->funcs, &X); A = tSqlExprCreateFunction(NULL, &X, &Y, X.type); }
// is (not) null expression
expr(A) ::= expr(X) IS NULL. {A = tSqlExprCreate(X, NULL, TK_ISNULL);}
expr(A) ::= expr(X) IS NOT NULL. {A = tSqlExprCreate(X, NULL, TK_NOTNULL);}
// relational expression
expr(A) ::= expr(X) LT expr(Y). {A = tSqlExprCreate(X, Y, TK_LT);}
expr(A) ::= expr(X) GT expr(Y). {A = tSqlExprCreate(X, Y, TK_GT);}
expr(A) ::= expr(X) LE expr(Y). {A = tSqlExprCreate(X, Y, TK_LE);}
expr(A) ::= expr(X) GE expr(Y). {A = tSqlExprCreate(X, Y, TK_GE);}
expr(A) ::= expr(X) NE expr(Y). {A = tSqlExprCreate(X, Y, TK_NE);}
expr(A) ::= expr(X) EQ expr(Y). {A = tSqlExprCreate(X, Y, TK_EQ);}
expr(A) ::= expr(X) BETWEEN expr(Y) AND expr(Z). { tSqlExpr* X2 = tSqlExprClone(X); A = tSqlExprCreate(tSqlExprCreate(X, Y, TK_GE), tSqlExprCreate(X2, Z, TK_LE), TK_AND);}
expr(A) ::= expr(X) AND expr(Y). {A = tSqlExprCreate(X, Y, TK_AND);}
expr(A) ::= expr(X) OR expr(Y). {A = tSqlExprCreate(X, Y, TK_OR); }
// binary arithmetic expression
expr(A) ::= expr(X) PLUS expr(Y). {A = tSqlExprCreate(X, Y, TK_PLUS); }
expr(A) ::= expr(X) MINUS expr(Y). {A = tSqlExprCreate(X, Y, TK_MINUS); }
expr(A) ::= expr(X) STAR expr(Y). {A = tSqlExprCreate(X, Y, TK_STAR); }
expr(A) ::= expr(X) SLASH expr(Y). {A = tSqlExprCreate(X, Y, TK_DIVIDE);}
expr(A) ::= expr(X) REM expr(Y). {A = tSqlExprCreate(X, Y, TK_REM); }
// like expression
expr(A) ::= expr(X) LIKE expr(Y). {A = tSqlExprCreate(X, Y, TK_LIKE); }
// match expression
expr(A) ::= expr(X) MATCH expr(Y). {A = tSqlExprCreate(X, Y, TK_MATCH); }
expr(A) ::= expr(X) NMATCH expr(Y). {A = tSqlExprCreate(X, Y, TK_NMATCH); }
//in expression
expr(A) ::= expr(X) IN LP exprlist(Y) RP. {A = tSqlExprCreate(X, (tSqlExpr*)Y, TK_IN); }
%type exprlist {SArray*}
%destructor exprlist {tSqlExprListDestroy($$);}
%type expritem {tSqlExpr*}
%destructor expritem {tSqlExprDestroy($$);}
exprlist(A) ::= exprlist(X) COMMA expritem(Y). {A = tSqlExprListAppend(X,Y,0, 0);}
exprlist(A) ::= expritem(X). {A = tSqlExprListAppend(0,X,0, 0);}
expritem(A) ::= expr(X). {A = X;}
expritem(A) ::= . {A = 0;}
///////////////////////////////////reset query cache//////////////////////////////////////
cmd ::= RESET QUERY CACHE. { setDCLSqlElems(pInfo, TSDB_SQL_RESET_CACHE, 0);}
///////////////////////////////////sync replica database//////////////////////////////////
cmd ::= SYNCDB ids(X) REPLICA.{ setDCLSqlElems(pInfo, TSDB_SQL_SYNC_DB_REPLICA, 1, &X);}
///////////////////////////////////ALTER TABLE statement//////////////////////////////////
cmd ::= ALTER TABLE ids(X) cpxName(F) ADD COLUMN columnlist(A). {
X.n += F.n;
SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, -1);
setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
}
cmd ::= ALTER TABLE ids(X) cpxName(F) DROP COLUMN ids(A). {
X.n += F.n;
toTSDBType(A.type);
SArray* K = tVariantListAppendToken(NULL, &A, -1);
SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN, -1);
setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
}
cmd ::= ALTER TABLE ids(X) cpxName(F) MODIFY COLUMN columnlist(A). {
X.n += F.n;
SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_CHANGE_COLUMN, -1);
setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
}
//////////////////////////////////ALTER TAGS statement/////////////////////////////////////
cmd ::= ALTER TABLE ids(X) cpxName(Y) ADD TAG columnlist(A). {
X.n += Y.n;
SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, -1);
setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
}
cmd ::= ALTER TABLE ids(X) cpxName(Z) DROP TAG ids(Y). {
X.n += Z.n;
toTSDBType(Y.type);
SArray* A = tVariantListAppendToken(NULL, &Y, -1);
SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN, -1);
setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
}
cmd ::= ALTER TABLE ids(X) cpxName(F) CHANGE TAG ids(Y) ids(Z). {
X.n += F.n;
toTSDBType(Y.type);
SArray* A = tVariantListAppendToken(NULL, &Y, -1);
toTSDBType(Z.type);
A = tVariantListAppendToken(A, &Z, -1);
SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN, -1);
setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
}
cmd ::= ALTER TABLE ids(X) cpxName(F) SET TAG ids(Y) EQ tagitem(Z). {
X.n += F.n;
toTSDBType(Y.type);
SArray* A = tVariantListAppendToken(NULL, &Y, -1);
A = tVariantListAppend(A, &Z, -1);
SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL, -1);
setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
}
cmd ::= ALTER TABLE ids(X) cpxName(F) MODIFY TAG columnlist(A). {
X.n += F.n;
SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_MODIFY_TAG_COLUMN, -1);
setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
}
///////////////////////////////////ALTER STABLE statement//////////////////////////////////
cmd ::= ALTER STABLE ids(X) cpxName(F) ADD COLUMN columnlist(A). {
X.n += F.n;
SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, TSDB_SUPER_TABLE);
setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
}
cmd ::= ALTER STABLE ids(X) cpxName(F) DROP COLUMN ids(A). {
X.n += F.n;
toTSDBType(A.type);
SArray* K = tVariantListAppendToken(NULL, &A, -1);
SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN, TSDB_SUPER_TABLE);
setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
}
cmd ::= ALTER STABLE ids(X) cpxName(F) MODIFY COLUMN columnlist(A). {
X.n += F.n;
SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_CHANGE_COLUMN, TSDB_SUPER_TABLE);
setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
}
//////////////////////////////////ALTER TAGS statement/////////////////////////////////////
cmd ::= ALTER STABLE ids(X) cpxName(Y) ADD TAG columnlist(A). {
X.n += Y.n;
SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, TSDB_SUPER_TABLE);
setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
}
cmd ::= ALTER STABLE ids(X) cpxName(Z) DROP TAG ids(Y). {
X.n += Z.n;
toTSDBType(Y.type);
SArray* A = tVariantListAppendToken(NULL, &Y, -1);
SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN, TSDB_SUPER_TABLE);
setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
}
cmd ::= ALTER STABLE ids(X) cpxName(F) CHANGE TAG ids(Y) ids(Z). {
X.n += F.n;
toTSDBType(Y.type);
SArray* A = tVariantListAppendToken(NULL, &Y, -1);
toTSDBType(Z.type);
A = tVariantListAppendToken(A, &Z, -1);
SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN, TSDB_SUPER_TABLE);
setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
}
cmd ::= ALTER STABLE ids(X) cpxName(F) SET TAG ids(Y) EQ tagitem(Z). {
X.n += F.n;
toTSDBType(Y.type);
SArray* A = tVariantListAppendToken(NULL, &Y, -1);
A = tVariantListAppend(A, &Z, -1);
SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL, TSDB_SUPER_TABLE);
setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
}
cmd ::= ALTER STABLE ids(X) cpxName(F) MODIFY TAG columnlist(A). {
X.n += F.n;
SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_MODIFY_TAG_COLUMN, TSDB_SUPER_TABLE);
setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
}
////////////////////////////////////////kill statement///////////////////////////////////////
cmd ::= KILL CONNECTION INTEGER(Y). {setKillSql(pInfo, TSDB_SQL_KILL_CONNECTION, &Y);}
cmd ::= KILL STREAM INTEGER(X) COLON(Z) INTEGER(Y). {X.n += (Z.n + Y.n); setKillSql(pInfo, TSDB_SQL_KILL_STREAM, &X);}
cmd ::= KILL QUERY INTEGER(X) COLON(Z) INTEGER(Y). {X.n += (Z.n + Y.n); setKillSql(pInfo, TSDB_SQL_KILL_QUERY, &X);}
%fallback ID ABORT AFTER ASC ATTACH BEFORE BEGIN CASCADE CLUSTER CONFLICT COPY DATABASE DEFERRED
DELIMITERS DESC DETACH EACH END EXPLAIN FAIL FOR GLOB IGNORE IMMEDIATE INITIALLY INSTEAD
LIKE MATCH NMATCH KEY OF OFFSET RAISE REPLACE RESTRICT ROW STATEMENT TRIGGER VIEW ALL
NOW IPTOKEN SEMI NONE PREV LINEAR IMPORT TBNAME JOIN STABLE NULL INSERT INTO VALUES.

4691
2.0/src/query/src/qAggMain.c
View File

File diff suppressed because it is too large Load Diff

8990
2.0/src/query/src/qExecutor.c
View File

File diff suppressed because it is too large Load Diff

1159
2.0/src/query/src/qExtbuffer.c
View File

File diff suppressed because it is too large Load Diff

523
2.0/src/query/src/qFill.c
View File

@ -1,523 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "qAggMain.h"
#include "taosdef.h"
#include "tmsg.h"
#include "ttype.h"
#include "qFill.h"
#include "qExtbuffer.h"
#include "queryLog.h"
#include "qExecutor.h"
#define FILL_IS_ASC_FILL(_f) ((_f)->order == TSDB_ORDER_ASC)
#define DO_INTERPOLATION(_v1, _v2, _k1, _k2, _k) ((_v1) + ((_v2) - (_v1)) * (((double)(_k)) - ((double)(_k1))) / (((double)(_k2)) - ((double)(_k1))))
static void setTagsValue(SFillInfo* pFillInfo, void** data, int32_t genRows) {
for(int32_t j = 0; j < pFillInfo->numOfCols; ++j) {
SFillColInfo* pCol = &pFillInfo->pFillCol[j];
if (TSDB_COL_IS_NORMAL_COL(pCol->flag) || TSDB_COL_IS_UD_COL(pCol->flag)) {
continue;
}
char* val1 = elePtrAt(data[j], pCol->col.bytes, genRows);
assert(pCol->tagIndex >= 0 && pCol->tagIndex < pFillInfo->numOfTags);
SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex];
assert (pTag->col.colId == pCol->col.colId);
assignVal(val1, pTag->tagVal, pCol->col.bytes, pCol->col.type);
}
}
static void setNullValueForRow(SFillInfo* pFillInfo, void** data, int32_t numOfCol, int32_t rowIndex) {
// the first are always the timestamp column, so start from the second column.
for (int32_t i = 1; i < numOfCol; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
char* output = elePtrAt(data[i], pCol->col.bytes, rowIndex);
setNull(output, pCol->col.type, pCol->col.bytes);
}
}
static void doFillOneRowResult(SFillInfo* pFillInfo, void** data, char** srcData, int64_t ts, bool outOfBound) {
char* prev = pFillInfo->prevValues;
char* next = pFillInfo->nextValues;
SPoint point1, point2, point;
int32_t step = GET_FORWARD_DIRECTION_FACTOR(pFillInfo->order);
// set the primary timestamp column value
int32_t index = pFillInfo->numOfCurrent;
char* val = elePtrAt(data[0], TSDB_KEYSIZE, index);
*(TSKEY*) val = pFillInfo->currentKey;
// set the other values
if (pFillInfo->type == TSDB_FILL_PREV) {
char* p = FILL_IS_ASC_FILL(pFillInfo) ? prev : next;
if (p != NULL) {
for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
if (TSDB_COL_IS_TAG(pCol->flag)) {
continue;
}
char* output = elePtrAt(data[i], pCol->col.bytes, index);
assignVal(output, p + pCol->col.offset, pCol->col.bytes, pCol->col.type);
}
} else { // no prev value yet, set the value for NULL
setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index);
}
} else if (pFillInfo->type == TSDB_FILL_NEXT) {
char* p = FILL_IS_ASC_FILL(pFillInfo)? next : prev;
if (p != NULL) {
for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
if (TSDB_COL_IS_TAG(pCol->flag)) {
continue;
}
char* output = elePtrAt(data[i], pCol->col.bytes, index);
assignVal(output, p + pCol->col.offset, pCol->col.bytes, pCol->col.type);
}
} else { // no prev value yet, set the value for NULL
setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index);
}
} else if (pFillInfo->type == TSDB_FILL_LINEAR) {
// TODO : linear interpolation supports NULL value
if (prev != NULL && !outOfBound) {
for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
if (TSDB_COL_IS_TAG(pCol->flag)) {
continue;
}
int16_t type = pCol->col.type;
int16_t bytes = pCol->col.bytes;
char *val1 = elePtrAt(data[i], pCol->col.bytes, index);
if (type == TSDB_DATA_TYPE_BINARY|| type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BOOL) {
setNull(val1, pCol->col.type, bytes);
continue;
}
point1 = (SPoint){.key = *(TSKEY*)(prev), .val = prev + pCol->col.offset};
point2 = (SPoint){.key = ts, .val = srcData[i] + pFillInfo->index * bytes};
point = (SPoint){.key = pFillInfo->currentKey, .val = val1};
taosGetLinearInterpolationVal(&point, type, &point1, &point2, type);
}
} else {
setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index);
}
} else { // fill the default value */
for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
if (TSDB_COL_IS_TAG(pCol->flag)/* || IS_VAR_DATA_TYPE(pCol->col.type)*/) {
continue;
}
char* val1 = elePtrAt(data[i], pCol->col.bytes, index);
assignVal(val1, (char*)&pCol->fillVal.i, pCol->col.bytes, pCol->col.type);
}
}
setTagsValue(pFillInfo, data, index);
pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pFillInfo->interval.sliding * step, pFillInfo->interval.slidingUnit, pFillInfo->precision);
pFillInfo->numOfCurrent++;
}
static void initBeforeAfterDataBuf(SFillInfo* pFillInfo, char** next) {
if (*next != NULL) {
return;
}
*next = calloc(1, pFillInfo->rowSize);
for (int i = 1; i < pFillInfo->numOfCols; i++) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
setNull(*next + pCol->col.offset, pCol->col.type, pCol->col.bytes);
}
}
static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, char** srcData, char* buf) {
int32_t rowIndex = pFillInfo->index;
for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
memcpy(buf + pCol->col.offset, srcData[i] + rowIndex * pCol->col.bytes, pCol->col.bytes);
}
}
static int32_t fillResultImpl(SFillInfo* pFillInfo, void** data, int32_t outputRows) {
pFillInfo->numOfCurrent = 0;
char** srcData = pFillInfo->pData;
char** prev = &pFillInfo->prevValues;
char** next = &pFillInfo->nextValues;
int32_t step = GET_FORWARD_DIRECTION_FACTOR(pFillInfo->order);
if (FILL_IS_ASC_FILL(pFillInfo)) {
assert(pFillInfo->currentKey >= pFillInfo->start);
} else {
assert(pFillInfo->currentKey <= pFillInfo->start);
}
while (pFillInfo->numOfCurrent < outputRows) {
int64_t ts = ((int64_t*)pFillInfo->pData[0])[pFillInfo->index];
// set the next value for interpolation
if ((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) ||
(pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) {
initBeforeAfterDataBuf(pFillInfo, next);
copyCurrentRowIntoBuf(pFillInfo, srcData, *next);
}
if (((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) || (pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) &&
pFillInfo->numOfCurrent < outputRows) {
// fill the gap between two actual input rows
while (((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) ||
(pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) &&
pFillInfo->numOfCurrent < outputRows) {
doFillOneRowResult(pFillInfo, data, srcData, ts, false);
}
// output buffer is full, abort
if (pFillInfo->numOfCurrent == outputRows) {
pFillInfo->numOfTotal += pFillInfo->numOfCurrent;
return outputRows;
}
} else {
assert(pFillInfo->currentKey == ts);
initBeforeAfterDataBuf(pFillInfo, prev);
if (pFillInfo->type == TSDB_FILL_NEXT && (pFillInfo->index + 1) < pFillInfo->numOfRows) {
initBeforeAfterDataBuf(pFillInfo, next);
++pFillInfo->index;
copyCurrentRowIntoBuf(pFillInfo, srcData, *next);
--pFillInfo->index;
}
// assign rows to dst buffer
for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
if (TSDB_COL_IS_TAG(pCol->flag)/* || IS_VAR_DATA_TYPE(pCol->col.type)*/) {
continue;
}
char* output = elePtrAt(data[i], pCol->col.bytes, pFillInfo->numOfCurrent);
char* src = elePtrAt(srcData[i], pCol->col.bytes, pFillInfo->index);
if (i == 0 || (pCol->functionId != TSDB_FUNC_COUNT && !isNull(src, pCol->col.type)) ||
(pCol->functionId == TSDB_FUNC_COUNT && GET_INT64_VAL(src) != 0)) {
assignVal(output, src, pCol->col.bytes, pCol->col.type);
memcpy(*prev + pCol->col.offset, src, pCol->col.bytes);
} else { // i > 0 and data is null , do interpolation
if (pFillInfo->type == TSDB_FILL_PREV) {
assignVal(output, *prev + pCol->col.offset, pCol->col.bytes, pCol->col.type);
} else if (pFillInfo->type == TSDB_FILL_LINEAR) {
assignVal(output, src, pCol->col.bytes, pCol->col.type);
memcpy(*prev + pCol->col.offset, src, pCol->col.bytes);
} else if (pFillInfo->type == TSDB_FILL_NEXT) {
if (*next) {
assignVal(output, *next + pCol->col.offset, pCol->col.bytes, pCol->col.type);
} else {
setNull(output, pCol->col.type, pCol->col.bytes);
}
} else {
assignVal(output, (char*)&pCol->fillVal.i, pCol->col.bytes, pCol->col.type);
}
}
}
// set the tag value for final result
setTagsValue(pFillInfo, data, pFillInfo->numOfCurrent);
pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pFillInfo->interval.sliding * step,
pFillInfo->interval.slidingUnit, pFillInfo->precision);
pFillInfo->index += 1;
pFillInfo->numOfCurrent += 1;
}
if (pFillInfo->index >= pFillInfo->numOfRows || pFillInfo->numOfCurrent >= outputRows) {
/* the raw data block is exhausted, next value does not exists */
if (pFillInfo->index >= pFillInfo->numOfRows) {
tfree(*next);
}
pFillInfo->numOfTotal += pFillInfo->numOfCurrent;
return pFillInfo->numOfCurrent;
}
}
return pFillInfo->numOfCurrent;
}
static int64_t appendFilledResult(SFillInfo* pFillInfo, void** output, int64_t resultCapacity) {
/*
* These data are generated according to fill strategy, since the current timestamp is out of the time window of
* real result set. Note that we need to keep the direct previous result rows, to generated the filled data.
*/
pFillInfo->numOfCurrent = 0;
while (pFillInfo->numOfCurrent < resultCapacity) {
doFillOneRowResult(pFillInfo, output, pFillInfo->pData, pFillInfo->start, true);
}
pFillInfo->numOfTotal += pFillInfo->numOfCurrent;
assert(pFillInfo->numOfCurrent == resultCapacity);
return resultCapacity;
}
// there are no duplicated tags in the SFillTagColInfo list
static int32_t setTagColumnInfo(SFillInfo* pFillInfo, int32_t numOfCols, int32_t capacity) {
int32_t rowsize = 0;
int32_t numOfTags = 0;
int32_t k = 0;
for (int32_t i = 0; i < numOfCols; ++i) {
SFillColInfo* pColInfo = &pFillInfo->pFillCol[i];
pFillInfo->pData[i] = NULL;
if (TSDB_COL_IS_TAG(pColInfo->flag) || pColInfo->col.type == TSDB_DATA_TYPE_BINARY) {
numOfTags += 1;
bool exists = false;
int32_t index = -1;
for (int32_t j = 0; j < k; ++j) {
if (pFillInfo->pTags[j].col.colId == pColInfo->col.colId) {
exists = true;
index = j;
break;
}
}
if (!exists) {
SSchema* pSchema = &pFillInfo->pTags[k].col;
pSchema->colId = pColInfo->col.colId;
pSchema->type = pColInfo->col.type;
pSchema->bytes = pColInfo->col.bytes;
pFillInfo->pTags[k].tagVal = calloc(1, pColInfo->col.bytes);
pColInfo->tagIndex = k;
k += 1;
} else {
pColInfo->tagIndex = index;
}
}
rowsize += pColInfo->col.bytes;
}
pFillInfo->numOfTags = numOfTags;
assert(k <= pFillInfo->numOfTags);
return rowsize;
}
static int32_t taosNumOfRemainRows(SFillInfo* pFillInfo) {
if (pFillInfo->numOfRows == 0 || (pFillInfo->numOfRows > 0 && pFillInfo->index >= pFillInfo->numOfRows)) {
return 0;
}
return pFillInfo->numOfRows - pFillInfo->index;
}
SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols,
int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType,
SFillColInfo* pCol, void* handle) {
if (fillType == TSDB_FILL_NONE) {
return NULL;
}
SFillInfo* pFillInfo = calloc(1, sizeof(SFillInfo));
taosResetFillInfo(pFillInfo, skey);
pFillInfo->order = order;
pFillInfo->type = fillType;
pFillInfo->pFillCol = pCol;
pFillInfo->numOfTags = numOfTags;
pFillInfo->numOfCols = numOfCols;
pFillInfo->precision = precision;
pFillInfo->alloc = capacity;
pFillInfo->handle = handle;
pFillInfo->interval.interval = slidingTime;
pFillInfo->interval.intervalUnit = slidingUnit;
pFillInfo->interval.sliding = slidingTime;
pFillInfo->interval.slidingUnit = slidingUnit;
pFillInfo->pData = malloc(POINTER_BYTES * numOfCols);
// if (numOfTags > 0) {
pFillInfo->pTags = calloc(numOfCols, sizeof(SFillTagColInfo));
for (int32_t i = 0; i < numOfCols; ++i) {
pFillInfo->pTags[i].col.colId = -2; // TODO
}
// }
pFillInfo->rowSize = setTagColumnInfo(pFillInfo, pFillInfo->numOfCols, pFillInfo->alloc);
assert(pFillInfo->rowSize > 0);
return pFillInfo;
}
void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp) {
pFillInfo->start = startTimestamp;
pFillInfo->currentKey = startTimestamp;
pFillInfo->end = startTimestamp;
pFillInfo->index = -1;
pFillInfo->numOfRows = 0;
pFillInfo->numOfCurrent = 0;
pFillInfo->numOfTotal = 0;
}
void* taosDestroyFillInfo(SFillInfo* pFillInfo) {
if (pFillInfo == NULL) {
return NULL;
}
tfree(pFillInfo->prevValues);
tfree(pFillInfo->nextValues);
for(int32_t i = 0; i < pFillInfo->numOfTags; ++i) {
tfree(pFillInfo->pTags[i].tagVal);
}
tfree(pFillInfo->pTags);
tfree(pFillInfo->pData);
tfree(pFillInfo->pFillCol);
tfree(pFillInfo);
return NULL;
}
void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) {
if (pFillInfo->type == TSDB_FILL_NONE) {
return;
}
pFillInfo->end = endKey;
if (!FILL_IS_ASC_FILL(pFillInfo)) {
pFillInfo->end = taosTimeTruncate(endKey, &pFillInfo->interval, pFillInfo->precision);
}
pFillInfo->index = 0;
pFillInfo->numOfRows = numOfRows;
}
void taosFillSetInputDataBlock(SFillInfo* pFillInfo, const SSDataBlock* pInput) {
for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
SColumnInfoData* pColData = taosArrayGet(pInput->pDataBlock, i);
pFillInfo->pData[i] = pColData->pData;
if (TSDB_COL_IS_TAG(pCol->flag)) { // copy the tag value to tag value buffer
SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex];
assert (pTag->col.colId == pCol->col.colId);
memcpy(pTag->tagVal, pColData->pData, pCol->col.bytes); // TODO not memcpy??
}
}
}
bool taosFillHasMoreResults(SFillInfo* pFillInfo) {
int32_t remain = taosNumOfRemainRows(pFillInfo);
if (remain > 0) {
return true;
}
if (pFillInfo->numOfTotal > 0 && (((pFillInfo->end > pFillInfo->start) && FILL_IS_ASC_FILL(pFillInfo)) ||
(pFillInfo->end < pFillInfo->start && !FILL_IS_ASC_FILL(pFillInfo)))) {
return getNumOfResultsAfterFillGap(pFillInfo, pFillInfo->end, 4096) > 0;
}
return false;
}
int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows) {
int64_t* tsList = (int64_t*) pFillInfo->pData[0];
int32_t numOfRows = taosNumOfRemainRows(pFillInfo);
TSKEY ekey1 = ekey;
if (!FILL_IS_ASC_FILL(pFillInfo)) {
pFillInfo->end = taosTimeTruncate(ekey, &pFillInfo->interval, pFillInfo->precision);
}
int64_t numOfRes = -1;
if (numOfRows > 0) { // still fill gap within current data block, not generating data after the result set.
TSKEY lastKey = tsList[pFillInfo->numOfRows - 1];
numOfRes = taosTimeCountInterval(
lastKey,
pFillInfo->currentKey,
pFillInfo->interval.sliding,
pFillInfo->interval.slidingUnit,
pFillInfo->precision);
numOfRes += 1;
assert(numOfRes >= numOfRows);
} else { // reach the end of data
if ((ekey1 < pFillInfo->currentKey && FILL_IS_ASC_FILL(pFillInfo)) ||
(ekey1 > pFillInfo->currentKey && !FILL_IS_ASC_FILL(pFillInfo))) {
return 0;
}
numOfRes = taosTimeCountInterval(
ekey1,
pFillInfo->currentKey,
pFillInfo->interval.sliding,
pFillInfo->interval.slidingUnit,
pFillInfo->precision);
numOfRes += 1;
}
return (numOfRes > maxNumOfRows) ? maxNumOfRows : numOfRes;
}
int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2, int32_t inputType) {
double v1 = -1, v2 = -1;
GET_TYPED_DATA(v1, double, inputType, point1->val);
GET_TYPED_DATA(v2, double, inputType, point2->val);
double r = DO_INTERPOLATION(v1, v2, point1->key, point2->key, point->key);
SET_TYPED_DATA(point->val, outputType, r);
return TSDB_CODE_SUCCESS;
}
int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, void** output, int32_t capacity) {
int32_t remain = taosNumOfRemainRows(pFillInfo);
int64_t numOfRes = getNumOfResultsAfterFillGap(pFillInfo, pFillInfo->end, capacity);
assert(numOfRes <= capacity);
// no data existed for fill operation now, append result according to the fill strategy
if (remain == 0) {
appendFilledResult(pFillInfo, output, numOfRes);
} else {
fillResultImpl(pFillInfo, output, (int32_t) numOfRes);
assert(numOfRes == pFillInfo->numOfCurrent);
}
qDebug("fill:%p, generated fill result, src block:%d, index:%d, brange:%"PRId64"-%"PRId64", currentKey:%"PRId64", current:%d, total:%d, %p",
pFillInfo, pFillInfo->numOfRows, pFillInfo->index, pFillInfo->start, pFillInfo->end, pFillInfo->currentKey, pFillInfo->numOfCurrent,
pFillInfo->numOfTotal, pFillInfo->handle);
return numOfRes;
}

3447
2.0/src/query/src/qFilter.c
View File

File diff suppressed because it is too large Load Diff

465
2.0/src/query/src/qFilterfunc.c
View File

@ -1,465 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "qExecutor.h"
#include "tmsg.h"
#include "tcompare.h"
#include "ttype.h"
#define FLT_COMPAR_TOL_FACTOR 4
#define FLT_EQUAL(_x, _y) (fabs((_x) - (_y)) <= (FLT_COMPAR_TOL_FACTOR * FLT_EPSILON))
#define FLT_GREATER(_x, _y) (!FLT_EQUAL((_x), (_y)) && ((_x) > (_y)))
#define FLT_LESS(_x, _y) (!FLT_EQUAL((_x), (_y)) && ((_x) < (_y)))
#define FLT_GREATEREQUAL(_x, _y) (FLT_EQUAL((_x), (_y)) || ((_x) > (_y)))
#define FLT_LESSEQUAL(_x, _y) (FLT_EQUAL((_x), (_y)) || ((_x) < (_y)))
bool lessOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type) {
SColumnFilterInfo* pFilterInfo = &pFilter->filterInfo;
switch(type) {
case TSDB_DATA_TYPE_TINYINT: return (*(int8_t *)minval < pFilterInfo->upperBndi);
case TSDB_DATA_TYPE_UTINYINT: return (*(uint8_t *)minval < pFilterInfo->upperBndi);
case TSDB_DATA_TYPE_SMALLINT: return (*(int16_t *)minval < pFilterInfo->upperBndi);
case TSDB_DATA_TYPE_USMALLINT: return (*(uint16_t *)minval < pFilterInfo->upperBndi);
case TSDB_DATA_TYPE_INT: return (*(int32_t *)minval < pFilterInfo->upperBndi);
case TSDB_DATA_TYPE_UINT: return (*(uint32_t *)minval < pFilterInfo->upperBndi);
case TSDB_DATA_TYPE_TIMESTAMP:
case TSDB_DATA_TYPE_BIGINT: return (*(int64_t *)minval < pFilterInfo->upperBndi);
case TSDB_DATA_TYPE_UBIGINT: return (*(uint64_t *)minval < pFilterInfo->upperBndi);
case TSDB_DATA_TYPE_FLOAT: return FLT_LESS(*(float*)minval, pFilter->filterInfo.upperBndd);
case TSDB_DATA_TYPE_DOUBLE: return (*(double *)minval < pFilterInfo->upperBndd);
default:
return false;
}
}
bool greaterOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type) {
SColumnFilterInfo *pFilterInfo = &pFilter->filterInfo;
switch (type) {
case TSDB_DATA_TYPE_TINYINT: return (*(int8_t *)maxval > pFilterInfo->lowerBndi);
case TSDB_DATA_TYPE_UTINYINT: return (*(uint8_t *)maxval > pFilterInfo->lowerBndi);
case TSDB_DATA_TYPE_SMALLINT: return (*(int16_t *)maxval > pFilterInfo->lowerBndi);
case TSDB_DATA_TYPE_USMALLINT: return (*(uint16_t *)maxval > pFilterInfo->lowerBndi);
case TSDB_DATA_TYPE_INT: return (*(int32_t *)maxval > pFilterInfo->lowerBndi);
case TSDB_DATA_TYPE_UINT: return (*(uint32_t *)maxval > pFilterInfo->lowerBndi);
case TSDB_DATA_TYPE_TIMESTAMP:
case TSDB_DATA_TYPE_BIGINT: return (*(int64_t *)maxval > pFilterInfo->lowerBndi);
case TSDB_DATA_TYPE_UBIGINT: return (*(uint64_t *)maxval > pFilterInfo->lowerBndi);
case TSDB_DATA_TYPE_FLOAT: return FLT_GREATER(*(float *)maxval, pFilterInfo->lowerBndd);
case TSDB_DATA_TYPE_DOUBLE: return (*(double *)maxval > pFilterInfo->lowerBndd);
default:
return false;
}
}
bool lessEqualOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type) {
SColumnFilterInfo* pFilterInfo = &pFilter->filterInfo;
switch(type) {
case TSDB_DATA_TYPE_TINYINT: return (*(int8_t *)minval <= pFilterInfo->upperBndi);
case TSDB_DATA_TYPE_UTINYINT: return (*(uint8_t *)minval <= pFilterInfo->upperBndi);
case TSDB_DATA_TYPE_SMALLINT: return (*(int16_t *)minval <= pFilterInfo->upperBndi);
case TSDB_DATA_TYPE_USMALLINT: return (*(uint16_t *)minval <= pFilterInfo->upperBndi);
case TSDB_DATA_TYPE_INT: return (*(int32_t *)minval <= pFilterInfo->upperBndi);
case TSDB_DATA_TYPE_UINT: return (*(uint32_t *)minval <= pFilterInfo->upperBndi);
case TSDB_DATA_TYPE_TIMESTAMP:
case TSDB_DATA_TYPE_BIGINT: return (*(int64_t *)minval <= pFilterInfo->upperBndi);
case TSDB_DATA_TYPE_UBIGINT: return (*(uint64_t *)minval <= pFilterInfo->upperBndi);
case TSDB_DATA_TYPE_FLOAT: return FLT_LESSEQUAL(*(float*)minval, pFilterInfo->upperBndd);
case TSDB_DATA_TYPE_DOUBLE: {
if ((fabs(*(double*)minval) - pFilterInfo->upperBndd) <= 2 * DBL_EPSILON) {
return true;
}
return (*(double *)minval <= pFilterInfo->upperBndd);
}
default:
return false;
}
}
bool greaterEqualOperator(SColumnFilterElem *pFilter, const char *minval, const char *maxval, int16_t type) {
SColumnFilterInfo *pFilterInfo = &pFilter->filterInfo;
switch (type) {
case TSDB_DATA_TYPE_TINYINT: return (*(int8_t *)maxval >= pFilterInfo->lowerBndi);
case TSDB_DATA_TYPE_UTINYINT: return (*(uint8_t *)maxval >= pFilterInfo->lowerBndi);
case TSDB_DATA_TYPE_SMALLINT: return (*(int16_t *)maxval >= pFilterInfo->lowerBndi);
case TSDB_DATA_TYPE_USMALLINT: return (*(uint16_t *)maxval >= pFilterInfo->lowerBndi);
case TSDB_DATA_TYPE_INT: return (*(int32_t *)maxval >= pFilterInfo->lowerBndi);
case TSDB_DATA_TYPE_UINT: return (*(uint32_t *)maxval >= pFilterInfo->lowerBndi);
case TSDB_DATA_TYPE_TIMESTAMP:
case TSDB_DATA_TYPE_BIGINT: return (*(int64_t *)maxval >= pFilterInfo->lowerBndi);
case TSDB_DATA_TYPE_UBIGINT: return (*(uint64_t *)maxval >= pFilterInfo->lowerBndi);
case TSDB_DATA_TYPE_FLOAT: return FLT_GREATEREQUAL(*(float*)maxval, pFilterInfo->lowerBndd);
case TSDB_DATA_TYPE_DOUBLE: {
if (fabs(*(double *)maxval - pFilterInfo->lowerBndd) <= 2 * DBL_EPSILON) {
return true;
}
return (*(double *)maxval - pFilterInfo->lowerBndd > (2 * DBL_EPSILON));
}
default:
return false;
}
}
////////////////////////////////////////////////////////////////////////
bool equalOperator(SColumnFilterElem *pFilter, const char *minval, const char *maxval, int16_t type) {
SColumnFilterInfo *pFilterInfo = &pFilter->filterInfo;
if (IS_SIGNED_NUMERIC_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || type == TSDB_DATA_TYPE_TIMESTAMP) {
int64_t minv = -1, maxv = -1;
GET_TYPED_DATA(minv, int64_t, type, minval);
GET_TYPED_DATA(maxv, int64_t, type, maxval);
if (minv == maxv) {
return minv == pFilterInfo->lowerBndi;
} else {
assert(minv < maxv);
return minv <= pFilterInfo->lowerBndi && pFilterInfo->lowerBndi <= maxv;
}
} else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
uint64_t minv = 0, maxv = 0;
GET_TYPED_DATA(minv, uint64_t, type, minval);
GET_TYPED_DATA(maxv, uint64_t, type, maxval);
if (minv == maxv) {
return minv == pFilterInfo->lowerBndi;
} else {
assert(minv < maxv);
return minv <= pFilterInfo->lowerBndi && pFilterInfo->lowerBndi <= maxv;
}
} else if (IS_FLOAT_TYPE(type)) {
double minv = -1, maxv = -1;
GET_TYPED_DATA(minv, double, type, minval);
GET_TYPED_DATA(maxv, double, type, maxval);
if (minv == maxv) {
return FLT_EQUAL(minv, pFilterInfo->lowerBndd);
} else { // range filter
assert(minv < maxv);
return minv <= pFilterInfo->lowerBndd && pFilterInfo->lowerBndd <= maxv;
}
} else if (type == TSDB_DATA_TYPE_BINARY) {
// query condition string is greater than the max length of string, not qualified data
if (pFilterInfo->len != varDataLen(minval)) {
return false;
}
return strncmp((char *)pFilterInfo->pz, varDataVal(minval), varDataLen(minval)) == 0;
} else if (type == TSDB_DATA_TYPE_NCHAR) {
// query condition string is greater than the max length of string, not qualified data
if (pFilterInfo->len != varDataLen(minval)) {
return false;
}
return wcsncmp((wchar_t *)pFilterInfo->pz, varDataVal(minval), varDataLen(minval) / TSDB_NCHAR_SIZE) == 0;
} else {
return false;
}
}
////////////////////////////////////////////////////////////////
bool likeOperator(SColumnFilterElem *pFilter, const char *minval, const char *maxval, int16_t type) {
if (type == TSDB_DATA_TYPE_BINARY) {
SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER;
return patternMatch((char *)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval), &info) == TSDB_PATTERN_MATCH;
} else if (type == TSDB_DATA_TYPE_NCHAR) {
SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER;
return WCSPatternMatch((wchar_t*)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval)/TSDB_NCHAR_SIZE, &info) == TSDB_PATTERN_MATCH;
} else {
return false;
}
}
////////////////////////////////////////////////////////////////
/**
* If minval equals to maxval, it may serve as the one element filter,
* or all elements of an array are identical during pref-filter stage.
* Otherwise, it must be pre-filter of array list of elements.
*
* During pre-filter stage, if there is one element that locates in [minval, maxval],
* the filter function will return true.
*/
// TODO not equal need to refactor
bool notEqualOperator(SColumnFilterElem *pFilter, const char *minval, const char *maxval, int16_t type) {
SColumnFilterInfo *pFilterInfo = &pFilter->filterInfo;
if (IS_SIGNED_NUMERIC_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || type == TSDB_DATA_TYPE_TIMESTAMP) {
int64_t minv = -1, maxv = -1;
GET_TYPED_DATA(minv, int64_t, type, minval);
GET_TYPED_DATA(maxv, int64_t, type, maxval);
if (minv == maxv) {
return minv != pFilterInfo->lowerBndi;
}
return true;
} else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
uint64_t minv = 0, maxv = 0;
GET_TYPED_DATA(minv, uint64_t, type, minval);
GET_TYPED_DATA(maxv, uint64_t, type, maxval);
if (minv == maxv) {
return minv != pFilterInfo->lowerBndi;
}
return true;
} else if (IS_FLOAT_TYPE(type)) {
double minv = -1, maxv = -1;
GET_TYPED_DATA(minv, double, type, minval);
GET_TYPED_DATA(maxv, double, type, maxval);
if (minv == maxv) {
return !FLT_EQUAL(minv, pFilterInfo->lowerBndd);
}
return true;
} else if (type == TSDB_DATA_TYPE_BINARY) {
if (pFilterInfo->len != varDataLen(minval)) {
return true;
}
return strncmp((char *)pFilterInfo->pz, varDataVal(minval), varDataLen(minval)) != 0;
} else if (type == TSDB_DATA_TYPE_NCHAR) {
if (pFilterInfo->len != varDataLen(minval)) {
return true;
}
return wcsncmp((wchar_t *)pFilterInfo->pz, varDataVal(minval), varDataLen(minval)/TSDB_NCHAR_SIZE) != 0;
} else {
return false;
}
}
////////////////////////////////////////////////////////////////
// dummy filter, not used
bool isNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type) {
return true;
}
bool notNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type) {
return true;
}
bool inOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type) {
if (type == TSDB_DATA_TYPE_BOOL || IS_SIGNED_NUMERIC_TYPE(type) || type == TSDB_DATA_TYPE_TIMESTAMP) {
int64_t minv = -1, maxv = -1;
GET_TYPED_DATA(minv, int64_t, type, minval);
GET_TYPED_DATA(maxv, int64_t, type, maxval);
if (minv == maxv) {
return NULL != taosHashGet((SHashObj *)pFilter->q, (char *)&minv, sizeof(minv));
}
return false;
} else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
uint64_t minv = 0, maxv = 0;
GET_TYPED_DATA(minv, uint64_t, type, minval);
GET_TYPED_DATA(maxv, uint64_t, type, maxval);
if (minv == maxv) {
return NULL != taosHashGet((SHashObj *)pFilter->q, (char *)&minv, sizeof(minv));
}
return false;
}else if (type == TSDB_DATA_TYPE_DOUBLE || type == TSDB_DATA_TYPE_FLOAT) {
double v;
GET_TYPED_DATA(v, double, type, minval);
return NULL != taosHashGet((SHashObj *)pFilter->q, (char *)&v, sizeof(v));
} else if (type == TSDB_DATA_TYPE_BINARY) {
return NULL != taosHashGet((SHashObj *)pFilter->q, varDataVal(minval), varDataLen(minval));
} else if (type == TSDB_DATA_TYPE_NCHAR){
return NULL != taosHashGet((SHashObj *)pFilter->q, varDataVal(minval), varDataLen(minval));
}
return false;
}
///////////////////////////////////////////////////////////////////////////////
bool rangeFilter_ii(SColumnFilterElem *pFilter, const char *minval, const char *maxval, int16_t type) {
SColumnFilterInfo *pFilterInfo = &pFilter->filterInfo;
switch (type) {
case TSDB_DATA_TYPE_TINYINT:
return ((*(int8_t *)minval <= pFilterInfo->upperBndi) && (*(int8_t *)maxval >= pFilterInfo->lowerBndi));
case TSDB_DATA_TYPE_UTINYINT:
return ((*(uint8_t *)minval <= pFilterInfo->upperBndi) && (*(uint8_t *)maxval >= pFilterInfo->lowerBndi));
case TSDB_DATA_TYPE_SMALLINT:
return ((*(int16_t *)minval <= pFilterInfo->upperBndi) && (*(int16_t *)maxval >= pFilterInfo->lowerBndi));
case TSDB_DATA_TYPE_USMALLINT:
return ((*(uint16_t *)minval <= pFilterInfo->upperBndi) && (*(uint16_t *)maxval >= pFilterInfo->lowerBndi));
case TSDB_DATA_TYPE_INT:
return ((*(int32_t *)minval <= pFilterInfo->upperBndi) && (*(int32_t *)maxval >= pFilterInfo->lowerBndi));
case TSDB_DATA_TYPE_UINT:
return ((*(uint32_t *)minval <= pFilterInfo->upperBndi) && (*(uint32_t *)maxval >= pFilterInfo->lowerBndi));
case TSDB_DATA_TYPE_TIMESTAMP:
case TSDB_DATA_TYPE_BIGINT:
return ((*(int64_t *)minval <= pFilterInfo->upperBndi) && (*(int64_t *)maxval >= pFilterInfo->lowerBndi));
case TSDB_DATA_TYPE_UBIGINT:
return ((*(uint64_t *)minval <= pFilterInfo->upperBndi) && (*(uint64_t *)maxval >= pFilterInfo->lowerBndi));
case TSDB_DATA_TYPE_FLOAT:
return FLT_LESSEQUAL(*(float *)minval, pFilterInfo->upperBndd) &&
FLT_GREATEREQUAL(*(float *)maxval, pFilterInfo->lowerBndd);
case TSDB_DATA_TYPE_DOUBLE:
return (*(double *)minval <= pFilterInfo->upperBndd && *(double *)maxval >= pFilterInfo->lowerBndd);
default:
return false;
}
}
bool rangeFilter_ee(SColumnFilterElem *pFilter, const char *minval, const char *maxval, int16_t type) {
SColumnFilterInfo *pFilterInfo = &pFilter->filterInfo;
switch (type) {
case TSDB_DATA_TYPE_TINYINT:
return ((*(int8_t *)minval < pFilterInfo->upperBndi) && (*(int8_t *)maxval > pFilterInfo->lowerBndi));
case TSDB_DATA_TYPE_UTINYINT:
return ((*(uint8_t *)minval < pFilterInfo->upperBndi) && (*(uint8_t *)maxval > pFilterInfo->lowerBndi));
case TSDB_DATA_TYPE_SMALLINT:
return ((*(int16_t *)minval < pFilterInfo->upperBndi) && (*(int16_t *)maxval > pFilterInfo->lowerBndi));
case TSDB_DATA_TYPE_USMALLINT:
return ((*(uint16_t *)minval < pFilterInfo->upperBndi) && (*(uint16_t *)maxval > pFilterInfo->lowerBndi));
case TSDB_DATA_TYPE_INT:
return ((*(int32_t *)minval < pFilterInfo->upperBndi) && (*(int32_t *)maxval > pFilterInfo->lowerBndi));
case TSDB_DATA_TYPE_UINT:
return ((*(uint32_t *)minval < pFilterInfo->upperBndi) && (*(uint32_t *)maxval > pFilterInfo->lowerBndi));
case TSDB_DATA_TYPE_TIMESTAMP:
case TSDB_DATA_TYPE_BIGINT:
return ((*(int64_t *)minval < pFilterInfo->upperBndi) && (*(int64_t *)maxval > pFilterInfo->lowerBndi));
case TSDB_DATA_TYPE_UBIGINT:
return ((*(uint64_t *)minval < pFilterInfo->upperBndi) && (*(uint64_t *)maxval > pFilterInfo->lowerBndi));
case TSDB_DATA_TYPE_FLOAT:
return ((*(float *)minval < pFilterInfo->upperBndd) && (*(float *)maxval > pFilterInfo->lowerBndd));
case TSDB_DATA_TYPE_DOUBLE:
return ((*(double *)minval < pFilterInfo->upperBndd) && (*(double *)maxval > pFilterInfo->lowerBndd));
default:
return false;
}
}
bool rangeFilter_ie(SColumnFilterElem *pFilter, const char *minval, const char *maxval, int16_t type) {
SColumnFilterInfo *pFilterInfo = &pFilter->filterInfo;
switch (type) {
case TSDB_DATA_TYPE_TINYINT:
return ((*(int8_t *)minval < pFilterInfo->upperBndi) && (*(int8_t *)maxval >= pFilterInfo->lowerBndi));
case TSDB_DATA_TYPE_UTINYINT:
return ((*(uint8_t *)minval < pFilterInfo->upperBndi) && (*(uint8_t *)maxval >= pFilterInfo->lowerBndi));
case TSDB_DATA_TYPE_SMALLINT:
return ((*(int16_t *)minval < pFilterInfo->upperBndi) && (*(int16_t *)maxval >= pFilterInfo->lowerBndi));
case TSDB_DATA_TYPE_USMALLINT:
return ((*(uint16_t *)minval < pFilterInfo->upperBndi) && (*(uint16_t *)maxval >= pFilterInfo->lowerBndi));
case TSDB_DATA_TYPE_INT:
return ((*(int32_t *)minval < pFilterInfo->upperBndi) && (*(int32_t *)maxval >= pFilterInfo->lowerBndi));
case TSDB_DATA_TYPE_UINT:
return ((*(uint32_t *)minval < pFilterInfo->upperBndi) && (*(uint32_t *)maxval >= pFilterInfo->lowerBndi));
case TSDB_DATA_TYPE_TIMESTAMP:
case TSDB_DATA_TYPE_BIGINT:
return ((*(int64_t *)minval < pFilterInfo->upperBndi) && (*(int64_t *)maxval >= pFilterInfo->lowerBndi));
case TSDB_DATA_TYPE_UBIGINT:
return ((*(uint64_t *)minval < pFilterInfo->upperBndi) && (*(uint64_t *)maxval >= pFilterInfo->lowerBndi));
case TSDB_DATA_TYPE_FLOAT:
return ((*(float *)minval < pFilterInfo->upperBndd) && (*(float *)maxval >= pFilterInfo->lowerBndd));
case TSDB_DATA_TYPE_DOUBLE:
return ((*(double *)minval < pFilterInfo->upperBndd) && (*(double *)maxval >= pFilterInfo->lowerBndd));
default:
return false;
}
}
bool rangeFilter_ei(SColumnFilterElem *pFilter, const char *minval, const char *maxval, int16_t type) {
SColumnFilterInfo *pFilterInfo = &pFilter->filterInfo;
switch (type) {
case TSDB_DATA_TYPE_TINYINT:
return ((*(int8_t *)minval <= pFilterInfo->upperBndi) && (*(int8_t *)maxval > pFilterInfo->lowerBndi));
case TSDB_DATA_TYPE_UTINYINT:
return ((*(uint8_t *)minval <= pFilterInfo->upperBndi) && (*(uint8_t *)maxval > pFilterInfo->lowerBndi));
case TSDB_DATA_TYPE_SMALLINT:
return ((*(int16_t *)minval <= pFilterInfo->upperBndi) && (*(int16_t *)maxval > pFilterInfo->lowerBndi));
case TSDB_DATA_TYPE_USMALLINT:
return ((*(uint16_t *)minval <= pFilterInfo->upperBndi) && (*(uint16_t *)maxval > pFilterInfo->lowerBndi));
case TSDB_DATA_TYPE_INT:
return ((*(int32_t *)minval <= pFilterInfo->upperBndi) && (*(int32_t *)maxval > pFilterInfo->lowerBndi));
case TSDB_DATA_TYPE_UINT:
return ((*(uint32_t *)minval <= pFilterInfo->upperBndi) && (*(uint32_t *)maxval > pFilterInfo->lowerBndi));
case TSDB_DATA_TYPE_TIMESTAMP:
case TSDB_DATA_TYPE_BIGINT:
return ((*(int64_t *)minval <= pFilterInfo->upperBndi) && (*(int64_t *)maxval > pFilterInfo->lowerBndi));
case TSDB_DATA_TYPE_UBIGINT:
return ((*(uint64_t *)minval <= pFilterInfo->upperBndi) && (*(uint64_t *)maxval > pFilterInfo->lowerBndi));
case TSDB_DATA_TYPE_FLOAT:
return FLT_GREATER(*(float *)maxval, pFilterInfo->lowerBndd) &&
FLT_LESSEQUAL(*(float *)minval, pFilterInfo->upperBndd);
case TSDB_DATA_TYPE_DOUBLE:
return ((*(double *)minval <= pFilterInfo->upperBndd) && (*(double *)maxval > pFilterInfo->lowerBndd));
default:
return false;
}
}
////////////////////////////////////////////////////////////////////////////
bool (*filterOperators[])(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type) = {
NULL,
lessOperator,
greaterOperator,
equalOperator,
lessEqualOperator,
greaterEqualOperator,
notEqualOperator,
likeOperator,
isNullOperator,
notNullOperator,
inOperator,
};
bool (*rangeFilterOperators[])(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type) = {
NULL,
rangeFilter_ee,
rangeFilter_ie,
rangeFilter_ei,
rangeFilter_ii,
};
__filter_func_t getFilterOperator(int32_t lowerOptr, int32_t upperOptr) {
__filter_func_t funcFp = NULL;
if ((lowerOptr == TSDB_RELATION_GREATER_EQUAL || lowerOptr == TSDB_RELATION_GREATER) &&
(upperOptr == TSDB_RELATION_LESS_EQUAL || upperOptr == TSDB_RELATION_LESS)) {
if (lowerOptr == TSDB_RELATION_GREATER_EQUAL) {
if (upperOptr == TSDB_RELATION_LESS_EQUAL) {
funcFp = rangeFilterOperators[4];
} else {
funcFp = rangeFilterOperators[2];
}
} else {
if (upperOptr == TSDB_RELATION_LESS_EQUAL) {
funcFp = rangeFilterOperators[3];
} else {
funcFp = rangeFilterOperators[1];
}
}
} else { // set callback filter function
if (lowerOptr != TSDB_RELATION_INVALID) {
funcFp = filterOperators[lowerOptr];
// invalid filter condition: %d", pQInfo, type
if (upperOptr != TSDB_RELATION_INVALID) {
return NULL;
}
} else {
funcFp = filterOperators[upperOptr];
}
}
return funcFp;
}

667
2.0/src/query/src/qHistogram.c
View File

@ -1,667 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "qHistogram.h"
#include "taosdef.h"
#include "tmsg.h"
#include "tlosertree.h"
/**
*
* implement the histogram and percentile_approx based on the paper:
* Yael Ben-Haim, Elad Tom-Tov. A Streaming Parallel Decision Tree Algorithm,
* The Journal of Machine Learning Research.Volume 11, 3/1/2010 pp.849-872
* https://dl.acm.org/citation.cfm?id=1756034
*
* @data 2018-12-14
* @version 0.1
*
*/
// SHeapEntry* tHeapCreate(int32_t numOfEntries) {
// SHeapEntry* pEntry = calloc(1, sizeof(SHeapEntry)*(numOfEntries + 1));
// return pEntry;
//}
//
// int32_t tHeapPut(SHeapEntry* pEntry, int32_t maxSize, int32_t num, void*
// pData, double v) {
// pEntry[num].val = v;
// pEntry[num].pData = pData;
//
// return num;
//}
//
////min heap
// void tHeapAdjust(SHeapEntry* pEntry, int32_t index, int32_t len) {
// SHeapEntry* ptr = NULL;
//
// int32_t end = len - 1;
//
// SHeapEntry p1 = pEntry[index];
// int32_t next = index;
//
// for(int32_t i=index; i<=(end-1)/2; ) {
// int32_t lc = (i<<1) + 1;
// int32_t rc = (i+1) << 1;
//
// ptr = &pEntry[lc];
// next = lc;
//
// if (rc < len && (pEntry[lc].val > pEntry[rc].val)) {
// ptr = &pEntry[rc];
// next = rc;
// }
//
// if (p1.val < ptr->val) {
// next = i;
// break;
// }
// pEntry[i] = *ptr;
// tSkipListNode* pnode = (tSkipListNode*) pEntry[i].pData;
// if(pnode != NULL) {
// ((SHistBin*) pnode->pData)->index = i;
// }
//
// i = next;
// }
//
// pEntry[next] = p1;
//
// tSkipListNode* pnode = (tSkipListNode*) p1.pData;
// if (pnode != NULL) {
// ((SHistBin*) pnode->pData)->index = next;
// }
//}
//
// void tHeapSort(SHeapEntry* pEntry, int32_t len) {
// int32_t last = len/2 - 1;
//
// for(int32_t i=last; i >= 0; --i) {
// tHeapAdjust(pEntry, i, len);
// }
//}
// typedef struct SInsertSupporter {
// int32_t numOfEntries;
// tSkipList* pSkipList;
// SLoserTreeInfo* pTree;
//} SInsertSupporter;
//
// int32_t compare(const void* pleft, const void* pright, void* param) {
// SLoserTreeNode* left = (SLoserTreeNode*) pleft;
// SLoserTreeNode* right = (SLoserTreeNode *)pright;
//
// SInsertSupporter* pss = (SInsertSupporter*) param;
//
// tSkipListNode* pLeftNode = (tSkipListNode*) left->pData;
// tSkipListNode* pRightNode = (tSkipListNode*) right->pData;
//
// SHistBin* pLeftBin = (SHistBin*)pLeftNode->pData;
// SHistBin* pRightBin = (SHistBin*)pRightNode->pData;
//
// if (pLeftBin->delta == pRightBin->delta) {
// return 0;
// } else {
// return ((pLeftBin->delta < pRightBin->delta)? -1:1);
// }
//}
static int32_t histogramCreateBin(SHistogramInfo* pHisto, int32_t index, double val);
SHistogramInfo* tHistogramCreate(int32_t numOfEntries) {
/* need one redundant slot */
SHistogramInfo* pHisto = malloc(sizeof(SHistogramInfo) + sizeof(SHistBin) * (numOfEntries + 1));
#if !defined(USE_ARRAYLIST)
pHisto->pList = SSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_DOUBLE, sizeof(double));
SInsertSupporter* pss = malloc(sizeof(SInsertSupporter));
pss->numOfEntries = pHisto->maxEntries;
pss->pSkipList = pHisto->pList;
int32_t ret = tLoserTreeCreate1(&pHisto->pLoserTree, numOfEntries, pss, compare);
pss->pTree = pHisto->pLoserTree;
#endif
return tHistogramCreateFrom(pHisto, numOfEntries);
}
SHistogramInfo* tHistogramCreateFrom(void* pBuf, int32_t numOfBins) {
memset(pBuf, 0, sizeof(SHistogramInfo) + sizeof(SHistBin) * (numOfBins + 1));
SHistogramInfo* pHisto = (SHistogramInfo*)pBuf;
pHisto->elems = (SHistBin*)((char*)pBuf + sizeof(SHistogramInfo));
for(int32_t i = 0; i < numOfBins; ++i) {
pHisto->elems[i].val = -DBL_MAX;
}
pHisto->maxEntries = numOfBins;
pHisto->min = DBL_MAX;
pHisto->max = -DBL_MAX;
return pBuf;
}
int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) {
if (*pHisto == NULL) {
*pHisto = tHistogramCreate(MAX_HISTOGRAM_BIN);
}
#if defined(USE_ARRAYLIST)
int32_t idx = histoBinarySearch((*pHisto)->elems, (*pHisto)->numOfEntries, val);
assert(idx >= 0 && idx <= (*pHisto)->maxEntries && (*pHisto)->elems != NULL);
if ((*pHisto)->elems[idx].val == val && idx >= 0) {
(*pHisto)->elems[idx].num += 1;
if ((*pHisto)->numOfEntries == 0) {
(*pHisto)->numOfEntries += 1;
}
} else { /* insert a new slot */
if ((*pHisto)->numOfElems >= 1 && idx < (*pHisto)->numOfEntries) {
if (idx > 0) {
assert((*pHisto)->elems[idx - 1].val <= val);
} else {
assert((*pHisto)->elems[idx].val > val);
}
} else if ((*pHisto)->numOfElems > 0) {
assert((*pHisto)->elems[(*pHisto)->numOfEntries].val <= val);
}
histogramCreateBin(*pHisto, idx, val);
}
#else
tSkipListKey key = tSkipListCreateKey(TSDB_DATA_TYPE_DOUBLE, &val, tDataTypes[TSDB_DATA_TYPE_DOUBLE].nSize);
SHistBin* entry = calloc(1, sizeof(SHistBin));
entry->val = val;
tSkipListNode* pResNode = SSkipListPut((*pHisto)->pList, entry, &key, 0);
SHistBin* pEntry1 = (SHistBin*)pResNode->pData;
pEntry1->index = -1;
tSkipListNode* pLast = NULL;
if (pEntry1->num == 0) { /* it is a new node */
(*pHisto)->numOfEntries += 1;
pEntry1->num += 1;
/* number of entries reaches the upper limitation */
if (pResNode->pForward[0] != NULL) {
/* we need to update the last updated slot in loser tree*/
pEntry1->delta = ((SHistBin*)pResNode->pForward[0]->pData)->val - val;
if ((*pHisto)->ordered) {
int32_t lastIndex = (*pHisto)->maxIndex;
SLoserTreeInfo* pTree = (*pHisto)->pLoserTree;
(*pHisto)->pLoserTree->pNode[lastIndex + pTree->numOfEntries].pData = pResNode;
pEntry1->index = (*pHisto)->pLoserTree->pNode[lastIndex + pTree->numOfEntries].index;
// update the loser tree
if ((*pHisto)->ordered) {
tLoserTreeAdjust(pTree, pEntry1->index + pTree->numOfEntries);
}
tSkipListKey kx =
tSkipListCreateKey(TSDB_DATA_TYPE_DOUBLE, &(*pHisto)->max, tDataTypes[TSDB_DATA_TYPE_DOUBLE].nSize);
pLast = tSkipListGetOne((*pHisto)->pList, &kx);
}
} else {
/* this node located at the last position of the skiplist, we do not
* update the loser-tree */
pEntry1->delta = DBL_MAX;
pLast = pResNode;
}
if (pResNode->pBackward[0] != &(*pHisto)->pList->pHead) {
SHistBin* pPrevEntry = (SHistBin*)pResNode->pBackward[0]->pData;
pPrevEntry->delta = val - pPrevEntry->val;
SLoserTreeInfo* pTree = (*pHisto)->pLoserTree;
if ((*pHisto)->ordered) {
tLoserTreeAdjust(pTree, pPrevEntry->index + pTree->numOfEntries);
tLoserTreeDisplay(pTree);
}
}
if ((*pHisto)->numOfEntries >= (*pHisto)->maxEntries + 1) {
// set the right value for loser-tree
assert((*pHisto)->pLoserTree != NULL);
if (!(*pHisto)->ordered) {
SSkipListPrint((*pHisto)->pList, 1);
SLoserTreeInfo* pTree = (*pHisto)->pLoserTree;
tSkipListNode* pHead = (*pHisto)->pList->pHead.pForward[0];
tSkipListNode* p1 = pHead;
printf("\n");
while (p1 != NULL) {
printf("%f\t", ((SHistBin*)(p1->pData))->delta);
p1 = p1->pForward[0];
}
printf("\n");
/* last one in skiplist is ignored */
for (int32_t i = pTree->numOfEntries; i < pTree->totalEntries; ++i) {
pTree->pNode[i].pData = pHead;
pTree->pNode[i].index = i - pTree->numOfEntries;
SHistBin* pBin = (SHistBin*)pHead->pData;
pBin->index = pTree->pNode[i].index;
pHead = pHead->pForward[0];
}
pLast = pHead;
for (int32_t i = 0; i < pTree->numOfEntries; ++i) {
pTree->pNode[i].index = -1;
}
tLoserTreeDisplay(pTree);
for (int32_t i = pTree->totalEntries - 1; i >= pTree->numOfEntries; i--) {
tLoserTreeAdjust(pTree, i);
}
tLoserTreeDisplay(pTree);
(*pHisto)->ordered = true;
}
printf("delta is:%lf\n", pEntry1->delta);
SSkipListPrint((*pHisto)->pList, 1);
/* the chosen node */
tSkipListNode* pNode = (*pHisto)->pLoserTree->pNode[0].pData;
SHistBin* pEntry = (SHistBin*)pNode->pData;
tSkipListNode* pNext = pNode->pForward[0];
SHistBin* pNextEntry = (SHistBin*)pNext->pData;
assert(pNextEntry->val - pEntry->val == pEntry->delta);
double newVal = (pEntry->val * pEntry->num + pNextEntry->val * pNextEntry->num) / (pEntry->num + pNextEntry->num);
pEntry->val = newVal;
pNode->key.dKey = newVal;
pEntry->num = pEntry->num + pNextEntry->num;
// update delta value in current node
pEntry->delta = (pNextEntry->delta + pNextEntry->val) - pEntry->val;
// reset delta value in the previous node
SHistBin* pPrevEntry = (SHistBin*)pNode->pBackward[0]->pData;
if (pPrevEntry) {
pPrevEntry->delta = pEntry->val - pPrevEntry->val;
}
SLoserTreeInfo* pTree = (*pHisto)->pLoserTree;
if (pNextEntry->index != -1) {
(*pHisto)->maxIndex = pNextEntry->index;
// set the last element in skiplist, of which delta is FLT_MAX;
pTree->pNode[pNextEntry->index + pTree->numOfEntries].pData = pLast;
((SHistBin*)pLast->pData)->index = pNextEntry->index;
int32_t f = pTree->pNode[pNextEntry->index + pTree->numOfEntries].index;
printf("disappear index is:%d\n", f);
}
tLoserTreeAdjust(pTree, pEntry->index + pTree->numOfEntries);
// remove the next node in skiplist
tSkipListRemoveNode((*pHisto)->pList, pNext);
SSkipListPrint((*pHisto)->pList, 1);
tLoserTreeDisplay((*pHisto)->pLoserTree);
} else { // add to heap
if (pResNode->pForward[0] != NULL) {
pEntry1->delta = ((SHistBin*)pResNode->pForward[0]->pData)->val - val;
} else {
pEntry1->delta = DBL_MAX;
}
if (pResNode->pBackward[0] != &(*pHisto)->pList->pHead) {
SHistBin* pPrevEntry = (SHistBin*)pResNode->pBackward[0]->pData;
pEntry1->delta = val - pPrevEntry->val;
}
printf("delta is:%9lf\n", pEntry1->delta);
}
} else {
SHistBin* pEntry = (SHistBin*)pResNode->pData;
assert(pEntry->val == val);
pEntry->num += 1;
}
#endif
if (val > (*pHisto)->max) {
(*pHisto)->max = val;
}
if (val < (*pHisto)->min) {
(*pHisto)->min = val;
}
(*pHisto)->numOfElems += 1;
return 0;
}
int32_t histoBinarySearch(SHistBin* pEntry, int32_t len, double val) {
int32_t end = len - 1;
int32_t start = 0;
while (start <= end) {
int32_t mid = (end - start) / 2 + start;
if (pEntry[mid].val == val) {
return mid;
}
if (pEntry[mid].val < val) {
start = mid + 1;
} else {
end = mid - 1;
}
}
int32_t ret = start > end ? start : end;
if (ret < 0) {
return 0;
} else {
return ret;
}
}
static void histogramMergeImpl(SHistBin* pHistBin, int32_t* size) {
#if defined(USE_ARRAYLIST)
int32_t oldSize = *size;
double delta = DBL_MAX;
int32_t index = -1;
for (int32_t i = 1; i < oldSize; ++i) {
double d = pHistBin[i].val - pHistBin[i - 1].val;
if (d < delta) {
delta = d;
index = i - 1;
}
}
SHistBin* s1 = &pHistBin[index];
SHistBin* s2 = &pHistBin[index + 1];
double newVal = (s1->val * s1->num + s2->val * s2->num) / (s1->num + s2->num);
s1->val = newVal;
s1->num = s1->num + s2->num;
memmove(&pHistBin[index + 1], &pHistBin[index + 2], (oldSize - index - 2) * sizeof(SHistBin));
(*size) -= 1;
#endif
}
/* optimize this procedure */
int32_t histogramCreateBin(SHistogramInfo* pHisto, int32_t index, double val) {
#if defined(USE_ARRAYLIST)
int32_t remain = pHisto->numOfEntries - index;
if (remain > 0) {
memmove(&pHisto->elems[index + 1], &pHisto->elems[index], sizeof(SHistBin) * remain);
}
assert(index >= 0 && index <= pHisto->maxEntries);
pHisto->elems[index].num = 1;
pHisto->elems[index].val = val;
pHisto->numOfEntries += 1;
/* we need to merge the slot */
if (pHisto->numOfEntries == pHisto->maxEntries + 1) {
histogramMergeImpl(pHisto->elems, &pHisto->numOfEntries);
pHisto->elems[pHisto->maxEntries].val = 0;
pHisto->elems[pHisto->maxEntries].num = 0;
}
#endif
assert(pHisto->numOfEntries <= pHisto->maxEntries);
return 0;
}
void tHistogramDestroy(SHistogramInfo** pHisto) {
if (*pHisto == NULL) {
return;
}
free(*pHisto);
*pHisto = NULL;
}
void tHistogramPrint(SHistogramInfo* pHisto) {
printf("total entries: %d, elements: %"PRId64 "\n", pHisto->numOfEntries, pHisto->numOfElems);
#if defined(USE_ARRAYLIST)
for (int32_t i = 0; i < pHisto->numOfEntries; ++i) {
printf("%d: (%f, %" PRId64 ")\n", i + 1, pHisto->elems[i].val, pHisto->elems[i].num);
}
#else
tSkipListNode* pNode = pHisto->pList->pHead.pForward[0];
for (int32_t i = 0; i < pHisto->numOfEntries; ++i) {
SHistBin* pEntry = (SHistBin*)pNode->pData;
printf("%d: (%f, %" PRId64 ")\n", i + 1, pEntry->val, pEntry->num);
pNode = pNode->pForward[0];
}
#endif
}
/**
* Estimated number of points in the interval (−inf,b].
* @param pHisto
* @param v
*/
int64_t tHistogramSum(SHistogramInfo* pHisto, double v) {
#if defined(USE_ARRAYLIST)
int32_t slotIdx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, v);
if (pHisto->elems[slotIdx].val != v) {
slotIdx -= 1;
if (slotIdx < 0) {
slotIdx = 0;
assert(v <= pHisto->elems[slotIdx].val);
} else {
assert(v >= pHisto->elems[slotIdx].val);
if (slotIdx + 1 < pHisto->numOfEntries) {
assert(v < pHisto->elems[slotIdx + 1].val);
}
}
}
double m1 = (double)pHisto->elems[slotIdx].num;
double v1 = pHisto->elems[slotIdx].val;
double m2 = (double)pHisto->elems[slotIdx + 1].num;
double v2 = pHisto->elems[slotIdx + 1].val;
double estNum = m1 + (m2 - m1) * (v - v1) / (v2 - v1);
double s1 = (m1 + estNum) * (v - v1) / (2 * (v2 - v1));
for (int32_t i = 0; i < slotIdx; ++i) {
s1 += pHisto->elems[i].num;
}
s1 = s1 + m1 / 2;
return (int64_t)s1;
#endif
}
double* tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num) {
#if defined(USE_ARRAYLIST)
double* pVal = malloc(num * sizeof(double));
for (int32_t i = 0; i < num; ++i) {
double numOfElem = (ratio[i] / 100) * pHisto->numOfElems;
if (numOfElem == 0) {
pVal[i] = pHisto->min;
continue;
} else if (numOfElem <= pHisto->elems[0].num) {
pVal[i] = pHisto->elems[0].val;
continue;
} else if (numOfElem == pHisto->numOfElems) {
pVal[i] = pHisto->max;
continue;
}
int32_t j = 0;
int64_t total = 0;
while (j < pHisto->numOfEntries) {
total += pHisto->elems[j].num;
if (total <= numOfElem && total + pHisto->elems[j + 1].num > numOfElem) {
break;
}
j += 1;
}
assert(total <= numOfElem && total + pHisto->elems[j + 1].num > numOfElem);
double delta = numOfElem - total;
if (fabs(delta) < FLT_EPSILON) {
pVal[i] = pHisto->elems[j].val;
}
double start = (double)pHisto->elems[j].num;
double range = pHisto->elems[j + 1].num - start;
if (range == 0) {
pVal[i] = (pHisto->elems[j + 1].val - pHisto->elems[j].val) * delta / start + pHisto->elems[j].val;
} else {
double factor = (-2 * start + sqrt(4 * start * start - 4 * range * (-2 * delta))) / (2 * range);
pVal[i] = pHisto->elems[j].val + (pHisto->elems[j + 1].val - pHisto->elems[j].val) * factor;
}
}
#else
double* pVal = malloc(num * sizeof(double));
for (int32_t i = 0; i < num; ++i) {
double numOfElem = ratio[i] * pHisto->numOfElems;
tSkipListNode* pFirst = pHisto->pList->pHead.pForward[0];
SHistBin* pEntry = (SHistBin*)pFirst->pData;
if (numOfElem == 0) {
pVal[i] = pHisto->min;
printf("i/numofSlot: %f, v:%f, %f\n", ratio[i], numOfElem, pVal[i]);
continue;
} else if (numOfElem <= pEntry->num) {
pVal[i] = pEntry->val;
printf("i/numofSlot: %f, v:%f, %f\n", ratio[i], numOfElem, pVal[i]);
continue;
} else if (numOfElem == pHisto->numOfElems) {
pVal[i] = pHisto->max;
printf("i/numofSlot: %f, v:%f, %f\n", ratio[i], numOfElem, pVal[i]);
continue;
}
int32_t j = 0;
int64_t total = 0;
SHistBin* pPrev = pEntry;
while (j < pHisto->numOfEntries) {
if (total <= numOfElem && total + pEntry->num > numOfElem) {
break;
}
total += pEntry->num;
pPrev = pEntry;
pFirst = pFirst->pForward[0];
pEntry = (SHistBin*)pFirst->pData;
j += 1;
}
assert(total <= numOfElem && total + pEntry->num > numOfElem);
double delta = numOfElem - total;
if (fabs(delta) < FLT_EPSILON) {
// printf("i/numofSlot: %f, v:%f, %f\n",
// (double)i/numOfSlots, numOfElem, pHisto->elems[j].val);
pVal[i] = pPrev->val;
}
double start = pPrev->num;
double range = pEntry->num - start;
if (range == 0) {
pVal[i] = (pEntry->val - pPrev->val) * delta / start + pPrev->val;
} else {
double factor = (-2 * start + sqrt(4 * start * start - 4 * range * (-2 * delta))) / (2 * range);
pVal[i] = pPrev->val + (pEntry->val - pPrev->val) * factor;
}
// printf("i/numofSlot: %f, v:%f, %f\n", (double)i/numOfSlots,
// numOfElem, val);
}
#endif
return pVal;
}
SHistogramInfo* tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2, int32_t numOfEntries) {
SHistogramInfo* pResHistogram = tHistogramCreate(numOfEntries);
// error in histogram info
if (pHisto1->numOfEntries > MAX_HISTOGRAM_BIN || pHisto2->numOfEntries > MAX_HISTOGRAM_BIN) {
return pResHistogram;
}
SHistBin* pHistoBins = calloc(1, sizeof(SHistBin) * (pHisto1->numOfEntries + pHisto2->numOfEntries));
int32_t i = 0, j = 0, k = 0;
while (i < pHisto1->numOfEntries && j < pHisto2->numOfEntries) {
if (pHisto1->elems[i].val < pHisto2->elems[j].val) {
pHistoBins[k++] = pHisto1->elems[i++];
} else if (pHisto1->elems[i].val > pHisto2->elems[j].val) {
pHistoBins[k++] = pHisto2->elems[j++];
} else {
pHistoBins[k] = pHisto1->elems[i++];
pHistoBins[k++].num += pHisto2->elems[j++].num;
}
}
if (i < pHisto1->numOfEntries) {
int32_t remain = pHisto1->numOfEntries - i;
memcpy(&pHistoBins[k], &pHisto1->elems[i], sizeof(SHistBin) * remain);
k += remain;
}
if (j < pHisto2->numOfEntries) {
int32_t remain = pHisto2->numOfEntries - j;
memcpy(&pHistoBins[k], &pHisto2->elems[j], sizeof(SHistBin) * remain);
k += remain;
}
/* update other information */
pResHistogram->numOfElems = pHisto1->numOfElems + pHisto2->numOfElems;
pResHistogram->min = (pHisto1->min < pHisto2->min) ? pHisto1->min : pHisto2->min;
pResHistogram->max = (pHisto1->max > pHisto2->max) ? pHisto1->max : pHisto2->max;
while (k > numOfEntries) {
histogramMergeImpl(pHistoBins, &k);
}
pResHistogram->numOfEntries = k;
memcpy(pResHistogram->elems, pHistoBins, sizeof(SHistBin) * k);
free(pHistoBins);
return pResHistogram;
}

535
2.0/src/query/src/qPercentile.c
View File

@ -1,535 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "qPercentile.h"
#include "qResultbuf.h"
#include "queryLog.h"
#include "taosdef.h"
#include "tcompare.h"
#include "ttype.h"
#define DEFAULT_NUM_OF_SLOT 1024
int32_t getGroupId(int32_t numOfSlots, int32_t slotIndex, int32_t times) {
return (times * numOfSlots) + slotIndex;
}
static tFilePage *loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx) {
tFilePage *buffer = (tFilePage *)calloc(1, pMemBucket->bytes * pMemBucket->pSlots[slotIdx].info.size + sizeof(tFilePage));
int32_t groupId = getGroupId(pMemBucket->numOfSlots, slotIdx, pMemBucket->times);
SIDList list = getDataBufPagesIdList(pMemBucket->pBuffer, groupId);
int32_t offset = 0;
for(int32_t i = 0; i < list->size; ++i) {
SPageInfo* pgInfo = *(SPageInfo**) taosArrayGet(list, i);
tFilePage* pg = getResBufPage(pMemBucket->pBuffer, pgInfo->pageId);
memcpy(buffer->data + offset, pg->data, (size_t)(pg->num * pMemBucket->bytes));
offset += (int32_t)(pg->num * pMemBucket->bytes);
}
qsort(buffer->data, pMemBucket->pSlots[slotIdx].info.size, pMemBucket->bytes, pMemBucket->comparFn);
return buffer;
}
static void resetBoundingBox(MinMaxEntry* range, int32_t type) {
if (IS_SIGNED_NUMERIC_TYPE(type)) {
range->i64MaxVal = INT64_MIN;
range->i64MinVal = INT64_MAX;
} else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
range->u64MaxVal = 0;
range->u64MinVal = UINT64_MAX;
} else {
range->dMaxVal = -DBL_MAX;
range->dMinVal = DBL_MAX;
}
}
static int32_t setBoundingBox(MinMaxEntry* range, int16_t type, double minval, double maxval) {
if (minval > maxval) {
return -1;
}
if (IS_SIGNED_NUMERIC_TYPE(type)) {
range->i64MinVal = (int64_t) minval;
range->i64MaxVal = (int64_t) maxval;
} else if (IS_UNSIGNED_NUMERIC_TYPE(type)){
range->u64MinVal = (uint64_t) minval;
range->u64MaxVal = (uint64_t) maxval;
} else {
range->dMinVal = minval;
range->dMaxVal = maxval;
}
return 0;
}
static void resetPosInfo(SSlotInfo* pInfo) {
pInfo->size = 0;
pInfo->pageId = -1;
pInfo->data = NULL;
}
double findOnlyResult(tMemBucket *pMemBucket) {
assert(pMemBucket->total == 1);
for (int32_t i = 0; i < pMemBucket->numOfSlots; ++i) {
tMemBucketSlot *pSlot = &pMemBucket->pSlots[i];
if (pSlot->info.size == 0) {
continue;
}
int32_t groupId = getGroupId(pMemBucket->numOfSlots, i, pMemBucket->times);
SIDList list = getDataBufPagesIdList(pMemBucket->pBuffer, groupId);
assert(list->size == 1);
SPageInfo* pgInfo = (SPageInfo*) taosArrayGetP(list, 0);
tFilePage* pPage = getResBufPage(pMemBucket->pBuffer, pgInfo->pageId);
assert(pPage->num == 1);
double v = 0;
GET_TYPED_DATA(v, double, pMemBucket->type, pPage->data);
return v;
}
return 0;
}
int32_t tBucketIntHash(tMemBucket *pBucket, const void *value) {
int64_t v = 0;
GET_TYPED_DATA(v, int64_t, pBucket->type, value);
int32_t index = -1;
if (v > pBucket->range.i64MaxVal || v < pBucket->range.i64MinVal) {
return index;
}
// divide the value range into 1024 buckets
uint64_t span = pBucket->range.i64MaxVal - pBucket->range.i64MinVal;
if (span < pBucket->numOfSlots) {
int64_t delta = v - pBucket->range.i64MinVal;
index = (delta % pBucket->numOfSlots);
} else {
double slotSpan = (double)span / pBucket->numOfSlots;
index = (int32_t)((v - pBucket->range.i64MinVal) / slotSpan);
if (v == pBucket->range.i64MaxVal) {
index -= 1;
}
}
assert(index >= 0 && index < pBucket->numOfSlots);
return index;
}
int32_t tBucketUintHash(tMemBucket *pBucket, const void *value) {
int64_t v = 0;
GET_TYPED_DATA(v, uint64_t, pBucket->type, value);
int32_t index = -1;
if (v > pBucket->range.u64MaxVal || v < pBucket->range.u64MinVal) {
return index;
}
// divide the value range into 1024 buckets
uint64_t span = pBucket->range.u64MaxVal - pBucket->range.u64MinVal;
if (span < pBucket->numOfSlots) {
int64_t delta = v - pBucket->range.u64MinVal;
index = (int32_t) (delta % pBucket->numOfSlots);
} else {
double slotSpan = (double)span / pBucket->numOfSlots;
index = (int32_t)((v - pBucket->range.u64MinVal) / slotSpan);
if (v == pBucket->range.u64MaxVal) {
index -= 1;
}
}
assert(index >= 0 && index < pBucket->numOfSlots);
return index;
}
int32_t tBucketDoubleHash(tMemBucket *pBucket, const void *value) {
double v = 0;
if (pBucket->type == TSDB_DATA_TYPE_FLOAT) {
v = GET_FLOAT_VAL(value);
} else {
v = GET_DOUBLE_VAL(value);
}
int32_t index = -1;
if (v > pBucket->range.dMaxVal || v < pBucket->range.dMinVal) {
return index;
}
// divide a range of [dMinVal, dMaxVal] into 1024 buckets
double span = pBucket->range.dMaxVal - pBucket->range.dMinVal;
if (span < pBucket->numOfSlots) {
int32_t delta = (int32_t)(v - pBucket->range.dMinVal);
index = (delta % pBucket->numOfSlots);
} else {
double slotSpan = span / pBucket->numOfSlots;
index = (int32_t)((v - pBucket->range.dMinVal) / slotSpan);
if (v == pBucket->range.dMaxVal) {
index -= 1;
}
}
assert(index >= 0 && index < pBucket->numOfSlots);
return index;
}
static __perc_hash_func_t getHashFunc(int32_t type) {
if (IS_SIGNED_NUMERIC_TYPE(type)) {
return tBucketIntHash;
} else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
return tBucketUintHash;
} else {
return tBucketDoubleHash;
}
}
static void resetSlotInfo(tMemBucket* pBucket) {
for (int32_t i = 0; i < pBucket->numOfSlots; ++i) {
tMemBucketSlot* pSlot = &pBucket->pSlots[i];
resetBoundingBox(&pSlot->range, pBucket->type);
resetPosInfo(&pSlot->info);
}
}
tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, double maxval) {
tMemBucket *pBucket = (tMemBucket *)calloc(1, sizeof(tMemBucket));
if (pBucket == NULL) {
return NULL;
}
pBucket->numOfSlots = DEFAULT_NUM_OF_SLOT;
pBucket->bufPageSize = DEFAULT_PAGE_SIZE * 4; // 4k per page
pBucket->type = dataType;
pBucket->bytes = nElemSize;
pBucket->total = 0;
pBucket->times = 1;
pBucket->maxCapacity = 200000;
if (setBoundingBox(&pBucket->range, pBucket->type, minval, maxval) != 0) {
qError("MemBucket:%p, invalid value range: %f-%f", pBucket, minval, maxval);
free(pBucket);
return NULL;
}
pBucket->elemPerPage = (pBucket->bufPageSize - sizeof(tFilePage))/pBucket->bytes;
pBucket->comparFn = getKeyComparFunc(pBucket->type, TSDB_ORDER_ASC);
pBucket->hashFunc = getHashFunc(pBucket->type);
if (pBucket->hashFunc == NULL) {
qError("MemBucket:%p, not support data type %d, failed", pBucket, pBucket->type);
free(pBucket);
return NULL;
}
pBucket->pSlots = (tMemBucketSlot *)calloc(pBucket->numOfSlots, sizeof(tMemBucketSlot));
if (pBucket->pSlots == NULL) {
free(pBucket);
return NULL;
}
resetSlotInfo(pBucket);
int32_t ret = createDiskbasedResultBuffer(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, 1);
if (ret != TSDB_CODE_SUCCESS) {
tMemBucketDestroy(pBucket);
return NULL;
}
qDebug("MemBucket:%p, elem size:%d", pBucket, pBucket->bytes);
return pBucket;
}
void tMemBucketDestroy(tMemBucket *pBucket) {
if (pBucket == NULL) {
return;
}
destroyResultBuf(pBucket->pBuffer);
tfree(pBucket->pSlots);
tfree(pBucket);
}
void tMemBucketUpdateBoundingBox(MinMaxEntry *r, const char *data, int32_t dataType) {
if (IS_SIGNED_NUMERIC_TYPE(dataType)) {
int64_t v = 0;
GET_TYPED_DATA(v, int64_t, dataType, data);
if (r->i64MinVal > v) {
r->i64MinVal = v;
}
if (r->i64MaxVal < v) {
r->i64MaxVal = v;
}
} else if (IS_UNSIGNED_NUMERIC_TYPE(dataType)) {
uint64_t v = 0;
GET_TYPED_DATA(v, uint64_t, dataType, data);
if (r->i64MinVal > v) {
r->i64MinVal = v;
}
if (r->i64MaxVal < v) {
r->i64MaxVal = v;
}
} else if (IS_FLOAT_TYPE(dataType)) {
double v = 0;
GET_TYPED_DATA(v, double, dataType, data);
if (r->dMinVal > v) {
r->dMinVal = v;
}
if (r->dMaxVal < v) {
r->dMaxVal = v;
}
} else {
assert(0);
}
}
/*
* in memory bucket, we only accept data array list
*/
int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) {
assert(pBucket != NULL && data != NULL && size > 0);
int32_t count = 0;
int32_t bytes = pBucket->bytes;
for (int32_t i = 0; i < size; ++i) {
char *d = (char *) data + i * bytes;
int32_t index = (pBucket->hashFunc)(pBucket, d);
if (index < 0) {
continue;
}
count += 1;
tMemBucketSlot *pSlot = &pBucket->pSlots[index];
tMemBucketUpdateBoundingBox(&pSlot->range, d, pBucket->type);
// ensure available memory pages to allocate
int32_t groupId = getGroupId(pBucket->numOfSlots, index, pBucket->times);
int32_t pageId = -1;
if (pSlot->info.data == NULL || pSlot->info.data->num >= pBucket->elemPerPage) {
if (pSlot->info.data != NULL) {
assert(pSlot->info.data->num >= pBucket->elemPerPage && pSlot->info.size > 0);
// keep the pointer in memory
releaseResBufPage(pBucket->pBuffer, pSlot->info.data);
pSlot->info.data = NULL;
}
pSlot->info.data = getNewDataBuf(pBucket->pBuffer, groupId, &pageId);
pSlot->info.pageId = pageId;
}
memcpy(pSlot->info.data->data + pSlot->info.data->num * pBucket->bytes, d, pBucket->bytes);
pSlot->info.data->num += 1;
pSlot->info.size += 1;
}
pBucket->total += count;
return 0;
}
////////////////////////////////////////////////////////////////////////////////////////////
/*
*
* now, we need to find the minimum value of the next slot for
* interpolating the percentile value
* j is the last slot of current segment, we need to get the first
* slot of the next segment.
*/
static MinMaxEntry getMinMaxEntryOfNextSlotWithData(tMemBucket *pMemBucket, int32_t slotIdx) {
int32_t j = slotIdx + 1;
while (j < pMemBucket->numOfSlots && (pMemBucket->pSlots[j].info.size == 0)) {
++j;
}
assert(j < pMemBucket->numOfSlots);
return pMemBucket->pSlots[j].range;
}
static bool isIdenticalData(tMemBucket *pMemBucket, int32_t index);
static double getIdenticalDataVal(tMemBucket* pMemBucket, int32_t slotIndex) {
assert(isIdenticalData(pMemBucket, slotIndex));
tMemBucketSlot *pSlot = &pMemBucket->pSlots[slotIndex];
double finalResult = 0.0;
if (IS_SIGNED_NUMERIC_TYPE(pMemBucket->type)) {
finalResult = (double) pSlot->range.i64MinVal;
} else if (IS_UNSIGNED_NUMERIC_TYPE(pMemBucket->type)) {
finalResult = (double) pSlot->range.u64MinVal;
} else {
finalResult = (double) pSlot->range.dMinVal;
}
return finalResult;
}
double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction) {
int32_t num = 0;
for (int32_t i = 0; i < pMemBucket->numOfSlots; ++i) {
tMemBucketSlot *pSlot = &pMemBucket->pSlots[i];
if (pSlot->info.size == 0) {
continue;
}
// required value in current slot
if (num < (count + 1) && num + pSlot->info.size >= (count + 1)) {
if (pSlot->info.size + num == (count + 1)) {
/*
* now, we need to find the minimum value of the next slot for interpolating the percentile value
* j is the last slot of current segment, we need to get the first slot of the next segment.
*/
MinMaxEntry next = getMinMaxEntryOfNextSlotWithData(pMemBucket, i);
double maxOfThisSlot = 0;
double minOfNextSlot = 0;
if (IS_SIGNED_NUMERIC_TYPE(pMemBucket->type)) {
maxOfThisSlot = (double) pSlot->range.i64MaxVal;
minOfNextSlot = (double) next.i64MinVal;
} else if (IS_UNSIGNED_NUMERIC_TYPE(pMemBucket->type)) {
maxOfThisSlot = (double) pSlot->range.u64MaxVal;
minOfNextSlot = (double) next.u64MinVal;
} else {
maxOfThisSlot = (double) pSlot->range.dMaxVal;
minOfNextSlot = (double) next.dMinVal;
}
assert(minOfNextSlot > maxOfThisSlot);
double val = (1 - fraction) * maxOfThisSlot + fraction * minOfNextSlot;
return val;
}
if (pSlot->info.size <= pMemBucket->maxCapacity) {
// data in buffer and file are merged together to be processed.
tFilePage *buffer = loadDataFromFilePage(pMemBucket, i);
int32_t currentIdx = count - num;
char *thisVal = buffer->data + pMemBucket->bytes * currentIdx;
char *nextVal = thisVal + pMemBucket->bytes;
double td = 1.0, nd = 1.0;
GET_TYPED_DATA(td, double, pMemBucket->type, thisVal);
GET_TYPED_DATA(nd, double, pMemBucket->type, nextVal);
double val = (1 - fraction) * td + fraction * nd;
tfree(buffer);
return val;
} else { // incur a second round bucket split
if (isIdenticalData(pMemBucket, i)) {
return getIdenticalDataVal(pMemBucket, i);
}
// try next round
pMemBucket->times += 1;
qDebug("MemBucket:%p, start next round data bucketing, time:%d", pMemBucket, pMemBucket->times);
pMemBucket->range = pSlot->range;
pMemBucket->total = 0;
resetSlotInfo(pMemBucket);
int32_t groupId = getGroupId(pMemBucket->numOfSlots, i, pMemBucket->times - 1);
SIDList list = getDataBufPagesIdList(pMemBucket->pBuffer, groupId);
assert(list->size > 0);
for (int32_t f = 0; f < list->size; ++f) {
SPageInfo *pgInfo = *(SPageInfo **)taosArrayGet(list, f);
tFilePage *pg = getResBufPage(pMemBucket->pBuffer, pgInfo->pageId);
tMemBucketPut(pMemBucket, pg->data, (int32_t)pg->num);
releaseResBufPageInfo(pMemBucket->pBuffer, pgInfo);
}
return getPercentileImpl(pMemBucket, count - num, fraction);
}
} else {
num += pSlot->info.size;
}
}
return 0;
}
double getPercentile(tMemBucket *pMemBucket, double percent) {
if (pMemBucket->total == 0) {
return 0.0;
}
// if only one elements exists, return it
if (pMemBucket->total == 1) {
return findOnlyResult(pMemBucket);
}
percent = fabs(percent);
// find the min/max value, no need to scan all data in bucket
if (fabs(percent - 100.0) < DBL_EPSILON || (percent < DBL_EPSILON)) {
MinMaxEntry* pRange = &pMemBucket->range;
if (IS_SIGNED_NUMERIC_TYPE(pMemBucket->type)) {
double v = (double)(fabs(percent - 100) < DBL_EPSILON ? pRange->i64MaxVal : pRange->i64MinVal);
return v;
} else if (IS_UNSIGNED_NUMERIC_TYPE(pMemBucket->type)) {
double v = (double)(fabs(percent - 100) < DBL_EPSILON ? pRange->u64MaxVal : pRange->u64MinVal);
return v;
} else {
return fabs(percent - 100) < DBL_EPSILON? pRange->dMaxVal:pRange->dMinVal;
}
}
double percentVal = (percent * (pMemBucket->total - 1)) / ((double)100.0);
// do put data by using buckets
int32_t orderIdx = (int32_t)percentVal;
return getPercentileImpl(pMemBucket, orderIdx, percentVal - orderIdx);
}
/*
* check if data in one slot are all identical only need to compare with the bounding box
*/
bool isIdenticalData(tMemBucket *pMemBucket, int32_t index) {
tMemBucketSlot *pSeg = &pMemBucket->pSlots[index];
if (IS_FLOAT_TYPE(pMemBucket->type)) {
return fabs(pSeg->range.dMaxVal - pSeg->range.dMinVal) < DBL_EPSILON;
} else {
return pSeg->range.i64MinVal == pSeg->range.i64MaxVal;
}
}

714
2.0/src/query/src/qPlan.c
View File

@ -1,714 +0,0 @@
#include "os.h"
#include "qTableMeta.h"
#include "qPlan.h"
#include "qExecutor.h"
#include "qUtil.h"
#include "texpr.h"
#include "tscUtil.h"
#include "tsclient.h"
#define QNODE_TAGSCAN 1
#define QNODE_TABLESCAN 2
#define QNODE_PROJECT 3
#define QNODE_AGGREGATE 4
#define QNODE_GROUPBY 5
#define QNODE_LIMIT 6
#define QNODE_JOIN 7
#define QNODE_DISTINCT 8
#define QNODE_SORT 9
#define QNODE_UNIONALL 10
#define QNODE_TIMEWINDOW 11
#define QNODE_SESSIONWINDOW 12
#define QNODE_FILL 13
typedef struct SFillEssInfo {
int32_t fillType; // fill type
int64_t *val; // fill value
} SFillEssInfo;
typedef struct SJoinCond {
bool tagExists; // denote if tag condition exists or not
SColumn *tagCond[2];
SColumn *colCond[2];
} SJoinCond;
static SQueryNode* createQueryNode(int32_t type, const char* name, SQueryNode** prev, int32_t numOfPrev,
SExprInfo** pExpr, int32_t numOfOutput, SQueryTableInfo* pTableInfo,
void* pExtInfo) {
SQueryNode* pNode = calloc(1, sizeof(SQueryNode));
pNode->info.type = type;
pNode->info.name = strdup(name);
if (pTableInfo->id.uid != 0 && pTableInfo->tableName) { // it is a true table
pNode->tableInfo.id = pTableInfo->id;
pNode->tableInfo.tableName = strdup(pTableInfo->tableName);
}
pNode->numOfOutput = numOfOutput;
pNode->pExpr = calloc(numOfOutput, sizeof(SExprInfo));
for(int32_t i = 0; i < numOfOutput; ++i) {
tscExprAssign(&pNode->pExpr[i], pExpr[i]);
}
pNode->pPrevNodes = taosArrayInit(4, POINTER_BYTES);
for(int32_t i = 0; i < numOfPrev; ++i) {
taosArrayPush(pNode->pPrevNodes, &prev[i]);
}
switch(type) {
case QNODE_TABLESCAN: {
STimeWindow* window = calloc(1, sizeof(STimeWindow));
memcpy(window, pExtInfo, sizeof(STimeWindow));
pNode->pExtInfo = window;
break;
}
case QNODE_TIMEWINDOW: {
SInterval* pInterval = calloc(1, sizeof(SInterval));
pNode->pExtInfo = pInterval;
memcpy(pInterval, pExtInfo, sizeof(SInterval));
break;
}
case QNODE_GROUPBY: {
SGroupbyExpr* p = (SGroupbyExpr*) pExtInfo;
SGroupbyExpr* pGroupbyExpr = calloc(1, sizeof(SGroupbyExpr));
pGroupbyExpr->tableIndex = p->tableIndex;
pGroupbyExpr->orderType = p->orderType;
pGroupbyExpr->orderIndex = p->orderIndex;
pGroupbyExpr->numOfGroupCols = p->numOfGroupCols;
pGroupbyExpr->columnInfo = taosArrayDup(p->columnInfo);
pNode->pExtInfo = pGroupbyExpr;
break;
}
case QNODE_FILL: { // todo !!
pNode->pExtInfo = pExtInfo;
break;
}
case QNODE_LIMIT: {
pNode->pExtInfo = calloc(1, sizeof(SLimitVal));
memcpy(pNode->pExtInfo, pExtInfo, sizeof(SLimitVal));
break;
}
}
return pNode;
}
static SQueryNode* doAddTableColumnNode(SQueryInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SQueryTableInfo* info,
SArray* pExprs, SArray* tableCols) {
if (pQueryInfo->onlyTagQuery) {
int32_t num = (int32_t) taosArrayGetSize(pExprs);
SQueryNode* pNode = createQueryNode(QNODE_TAGSCAN, "TableTagScan", NULL, 0, pExprs->pData, num, info, NULL);
if (pQueryInfo->distinct) {
pNode = createQueryNode(QNODE_DISTINCT, "Distinct", &pNode, 1, pExprs->pData, num, info, NULL);
}
return pNode;
}
STimeWindow* window = &pQueryInfo->window;
SQueryNode* pNode = createQueryNode(QNODE_TABLESCAN, "TableScan", NULL, 0, NULL, 0, info, window);
if (pQueryInfo->projectionQuery) {
int32_t numOfOutput = (int32_t) taosArrayGetSize(pExprs);
pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExprs->pData, numOfOutput, info, NULL);
} else {
// table source column projection, generate the projection expr
int32_t numOfCols = (int32_t) taosArrayGetSize(tableCols);
SExprInfo** pExpr = calloc(numOfCols, POINTER_BYTES);
SSchema* pSchema = pTableMetaInfo->pTableMeta->schema;
for (int32_t i = 0; i < numOfCols; ++i) {
SColumn* pCol = taosArrayGetP(tableCols, i);
SColumnIndex index = {.tableIndex = 0, .columnIndex = pCol->columnIndex};
STableMetaInfo* pTableMetaInfo1 = tscGetMetaInfo(pQueryInfo, index.tableIndex);
SExprInfo* p = tscExprCreate(pTableMetaInfo1, TSDB_FUNC_PRJ, &index, pCol->info.type, pCol->info.bytes,
pCol->info.colId, 0, TSDB_COL_NORMAL);
strncpy(p->base.aliasName, pSchema[pCol->columnIndex].name, tListLen(p->base.aliasName));
pExpr[i] = p;
}
pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExpr, numOfCols, info, NULL);
for (int32_t i = 0; i < numOfCols; ++i) {
destroyQueryFuncExpr(pExpr[i], 1);
}
tfree(pExpr);
}
return pNode;
}
static SQueryNode* doCreateQueryPlanForOneTableImpl(SQueryInfo* pQueryInfo, SQueryNode* pNode, SQueryTableInfo* info,
SArray* pExprs) {
// check for aggregation
if (pQueryInfo->interval.interval > 0) {
int32_t numOfOutput = (int32_t)taosArrayGetSize(pExprs);
pNode = createQueryNode(QNODE_TIMEWINDOW, "TimeWindowAgg", &pNode, 1, pExprs->pData, numOfOutput, info,
&pQueryInfo->interval);
if (pQueryInfo->groupbyExpr.numOfGroupCols != 0) {
pNode = createQueryNode(QNODE_GROUPBY, "Groupby", &pNode, 1, pExprs->pData, numOfOutput, info, &pQueryInfo->groupbyExpr);
}
} else if (pQueryInfo->groupbyColumn) {
int32_t numOfOutput = (int32_t)taosArrayGetSize(pExprs);
pNode = createQueryNode(QNODE_GROUPBY, "Groupby", &pNode, 1, pExprs->pData, numOfOutput, info,
&pQueryInfo->groupbyExpr);
} else if (pQueryInfo->sessionWindow.gap > 0) {
pNode = createQueryNode(QNODE_SESSIONWINDOW, "SessionWindowAgg", &pNode, 1, NULL, 0, info, NULL);
} else if (pQueryInfo->simpleAgg) {
int32_t numOfOutput = (int32_t)taosArrayGetSize(pExprs);
pNode = createQueryNode(QNODE_AGGREGATE, "Aggregate", &pNode, 1, pExprs->pData, numOfOutput, info, NULL);
}
if (pQueryInfo->havingFieldNum > 0 || pQueryInfo->arithmeticOnAgg) {
int32_t numOfExpr = (int32_t)taosArrayGetSize(pQueryInfo->exprList1);
pNode =
createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pQueryInfo->exprList1->pData, numOfExpr, info, NULL);
}
if (pQueryInfo->fillType != TSDB_FILL_NONE) {
SFillEssInfo* pInfo = calloc(1, sizeof(SFillEssInfo));
pInfo->fillType = pQueryInfo->fillType;
pInfo->val = calloc(pNode->numOfOutput, sizeof(int64_t));
memcpy(pInfo->val, pQueryInfo->fillVal, pNode->numOfOutput);
pNode = createQueryNode(QNODE_FILL, "Fill", &pNode, 1, NULL, 0, info, pInfo);
}
if (pQueryInfo->limit.limit != -1 || pQueryInfo->limit.offset != 0) {
pNode = createQueryNode(QNODE_LIMIT, "Limit", &pNode, 1, NULL, 0, info, &pQueryInfo->limit);
}
return pNode;
}
static SQueryNode* doCreateQueryPlanForSingleTable(SQueryInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SArray* pExprs,
SArray* tableCols) {
char name[TSDB_TABLE_FNAME_LEN] = {0};
tNameExtractFullName(&pTableMetaInfo->name, name);
SQueryTableInfo info = {.tableName = strdup(name), .id = pTableMetaInfo->pTableMeta->id,};
// handle the only tag query
SQueryNode* pNode = doAddTableColumnNode(pQueryInfo, pTableMetaInfo, &info, pExprs, tableCols);
if (pQueryInfo->onlyTagQuery) {
tfree(info.tableName);
return pNode;
}
SQueryNode* pNode1 = doCreateQueryPlanForOneTableImpl(pQueryInfo, pNode, &info, pExprs);
tfree(info.tableName);
return pNode1;
}
SArray* createQueryPlanImpl(SQueryInfo* pQueryInfo) {
SArray* upstream = NULL;
if (pQueryInfo->pUpstream != NULL && taosArrayGetSize(pQueryInfo->pUpstream) > 0) { // subquery in the from clause
upstream = taosArrayInit(4, POINTER_BYTES);
size_t size = taosArrayGetSize(pQueryInfo->pUpstream);
for(int32_t i = 0; i < size; ++i) {
SQueryInfo* pq = taosArrayGet(pQueryInfo->pUpstream, i);
SArray* p = createQueryPlanImpl(pq);
taosArrayAddBatch(upstream, p->pData, (int32_t) taosArrayGetSize(p));
}
}
if (pQueryInfo->numOfTables > 1) { // it is a join query
// 1. separate the select clause according to table
taosArrayDestroy(upstream);
upstream = taosArrayInit(5, POINTER_BYTES);
for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[i];
uint64_t uid = pTableMetaInfo->pTableMeta->id.uid;
SArray* exprList = taosArrayInit(4, POINTER_BYTES);
if (tscExprCopy(exprList, pQueryInfo->exprList, uid, true) != 0) {
terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
tscExprDestroy(exprList);
exit(-1);
}
// 2. create the query execution node
char name[TSDB_TABLE_FNAME_LEN] = {0};
tNameExtractFullName(&pTableMetaInfo->name, name);
SQueryTableInfo info = {.tableName = strdup(name), .id = pTableMetaInfo->pTableMeta->id,};
// 3. get the required table column list
SArray* tableColumnList = taosArrayInit(4, sizeof(SColumn));
tscColumnListCopy(tableColumnList, pQueryInfo->colList, uid);
// 4. add the projection query node
SQueryNode* pNode = doAddTableColumnNode(pQueryInfo, pTableMetaInfo, &info, exprList, tableColumnList);
tscColumnListDestroy(tableColumnList);
tscExprDestroy(exprList);
taosArrayPush(upstream, &pNode);
}
// 3. add the join node here
SQueryTableInfo info = {0};
int32_t num = (int32_t) taosArrayGetSize(pQueryInfo->exprList);
SQueryNode* pNode = createQueryNode(QNODE_JOIN, "Join", upstream->pData, pQueryInfo->numOfTables,
pQueryInfo->exprList->pData, num, &info, NULL);
// 4. add the aggregation or projection execution node
pNode = doCreateQueryPlanForOneTableImpl(pQueryInfo, pNode, &info, pQueryInfo->exprList);
upstream = taosArrayInit(5, POINTER_BYTES);
taosArrayPush(upstream, &pNode);
} else { // only one table, normal query process
STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[0];
SQueryNode* pNode = doCreateQueryPlanForSingleTable(pQueryInfo, pTableMetaInfo, pQueryInfo->exprList, pQueryInfo->colList);
upstream = taosArrayInit(5, POINTER_BYTES);
taosArrayPush(upstream, &pNode);
}
return upstream;
}
SQueryNode* qCreateQueryPlan(SQueryInfo* pQueryInfo) {
SArray* upstream = createQueryPlanImpl(pQueryInfo);
assert(taosArrayGetSize(upstream) == 1);
SQueryNode* p = taosArrayGetP(upstream, 0);
taosArrayDestroy(upstream);
return p;
}
static void doDestroyQueryNode(SQueryNode* pQueryNode) {
tfree(pQueryNode->pExtInfo);
tfree(pQueryNode->pSchema);
tfree(pQueryNode->info.name);
tfree(pQueryNode->tableInfo.tableName);
pQueryNode->pExpr = destroyQueryFuncExpr(pQueryNode->pExpr, pQueryNode->numOfOutput);
if (pQueryNode->pPrevNodes != NULL) {
int32_t size = (int32_t) taosArrayGetSize(pQueryNode->pPrevNodes);
for(int32_t i = 0; i < size; ++i) {
SQueryNode* p = taosArrayGetP(pQueryNode->pPrevNodes, i);
doDestroyQueryNode(p);
}
taosArrayDestroy(pQueryNode->pPrevNodes);
}
tfree(pQueryNode);
}
void* qDestroyQueryPlan(SQueryNode* pQueryNode) {
if (pQueryNode == NULL) {
return NULL;
}
doDestroyQueryNode(pQueryNode);
return NULL;
}
bool hasAliasName(SExprInfo* pExpr) {
assert(pExpr != NULL);
return strncmp(pExpr->base.token, pExpr->base.aliasName, tListLen(pExpr->base.aliasName)) != 0;
}
static int32_t doPrintPlan(char* buf, SQueryNode* pQueryNode, int32_t level, int32_t totalLen) {
if (level > 0) {
sprintf(buf + totalLen, "%*c", level, ' ');
totalLen += level;
}
int32_t len1 = sprintf(buf + totalLen, "%s(", pQueryNode->info.name);
int32_t len = len1 + totalLen;
switch(pQueryNode->info.type) {
case QNODE_TABLESCAN: {
STimeWindow* win = (STimeWindow*)pQueryNode->pExtInfo;
len1 = sprintf(buf + len, "%s #%" PRIu64 ") time_range: %" PRId64 " - %" PRId64 "\n",
pQueryNode->tableInfo.tableName, pQueryNode->tableInfo.id.uid, win->skey, win->ekey);
len += len1;
break;
}
case QNODE_PROJECT: {
len1 = sprintf(buf + len, "cols: ");
len += len1;
for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) {
SSqlExpr* p = &pQueryNode->pExpr[i].base;
len1 = sprintf(buf + len, "[%s #%d]", p->aliasName, p->resColId);
len += len1;
if (i < pQueryNode->numOfOutput - 1) {
len1 = sprintf(buf + len, ", ");
len += len1;
}
}
len1 = sprintf(buf + len, ")");
len += len1;
//todo print filter info
len1 = sprintf(buf + len, " filters:(nil)\n");
len += len1;
break;
}
case QNODE_AGGREGATE: {
for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) {
SSqlExpr* pExpr = &pQueryNode->pExpr[i].base;
if (hasAliasName(&pQueryNode->pExpr[i])) {
len1 = sprintf(buf + len,"[%s #%s]", pExpr->token, pExpr->aliasName);
} else {
len1 = sprintf(buf + len,"[%s]", pExpr->token);
}
len += len1;
if (i < pQueryNode->numOfOutput - 1) {
len1 = sprintf(buf + len, ", ");
len += len1;
}
}
len1 = sprintf(buf + len, ")\n");
len += len1;
break;
}
case QNODE_TIMEWINDOW: {
for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) {
SSqlExpr* pExpr = &pQueryNode->pExpr[i].base;
if (hasAliasName(&pQueryNode->pExpr[i])) {
len1 = sprintf(buf + len,"[%s #%s]", pExpr->token, pExpr->aliasName);
} else {
len1 = sprintf(buf + len,"[%s]", pExpr->token);
}
len += len1;
if (i < pQueryNode->numOfOutput - 1) {
len1 = sprintf(buf + len,", ");
len += len1;
}
}
len1 = sprintf(buf + len,") ");
len += len1;
SInterval* pInterval = pQueryNode->pExtInfo;
len1 = sprintf(buf + len, "interval:%" PRId64 "(%s), sliding:%" PRId64 "(%s), offset:%" PRId64 "\n",
pInterval->interval, TSDB_TIME_PRECISION_MILLI_STR, pInterval->sliding, TSDB_TIME_PRECISION_MILLI_STR,
pInterval->offset);
len += len1;
break;
}
case QNODE_GROUPBY: { // todo hide the invisible column
for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) {
SSqlExpr* pExpr = &pQueryNode->pExpr[i].base;
if (hasAliasName(&pQueryNode->pExpr[i])) {
len1 = sprintf(buf + len,"[%s #%s]", pExpr->token, pExpr->aliasName);
} else {
len1 = sprintf(buf + len,"[%s]", pExpr->token);
}
len += len1;
if (i < pQueryNode->numOfOutput - 1) {
len1 = sprintf(buf + len,", ");
len += len1;
}
}
SGroupbyExpr* pGroupbyExpr = pQueryNode->pExtInfo;
SColIndex* pIndex = taosArrayGet(pGroupbyExpr->columnInfo, 0);
len1 = sprintf(buf + len,") groupby_col: [%s #%d]\n", pIndex->name, pIndex->colId);
len += len1;
break;
}
case QNODE_FILL: {
SFillEssInfo* pEssInfo = pQueryNode->pExtInfo;
len1 = sprintf(buf + len,"%d", pEssInfo->fillType);
len += len1;
if (pEssInfo->fillType == TSDB_FILL_SET_VALUE) {
len1 = sprintf(buf + len,", val:");
len += len1;
// todo get the correct fill data type
for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) {
len1 = sprintf(buf + len,"%"PRId64, pEssInfo->val[i]);
len += len1;
if (i < pQueryNode->numOfOutput - 1) {
len1 = sprintf(buf + len,", ");
len += len1;
}
}
}
len1 = sprintf(buf + len,")\n");
len += len1;
break;
}
case QNODE_LIMIT: {
SLimitVal* pVal = pQueryNode->pExtInfo;
len1 = sprintf(buf + len,"limit: %"PRId64", offset: %"PRId64")\n", pVal->limit, pVal->offset);
len += len1;
break;
}
case QNODE_DISTINCT:
case QNODE_TAGSCAN: {
len1 = sprintf(buf + len,"cols: ");
len += len1;
for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) {
SSqlExpr* p = &pQueryNode->pExpr[i].base;
len1 = sprintf(buf + len,"[%s #%d]", p->aliasName, p->resColId);
len += len1;
if (i < pQueryNode->numOfOutput - 1) {
len1 = sprintf(buf + len,", ");
len += len1;
}
}
len1 = sprintf(buf + len,")\n");
len += len1;
break;
}
case QNODE_JOIN: {
// print join condition
len1 = sprintf(buf + len, ")\n");
len += len1;
break;
}
}
return len;
}
int32_t queryPlanToStringImpl(char* buf, SQueryNode* pQueryNode, int32_t level, int32_t totalLen) {
int32_t len = doPrintPlan(buf, pQueryNode, level, totalLen);
for(int32_t i = 0; i < taosArrayGetSize(pQueryNode->pPrevNodes); ++i) {
SQueryNode* p1 = taosArrayGetP(pQueryNode->pPrevNodes, i);
int32_t len1 = queryPlanToStringImpl(buf, p1, level + 1, len);
len = len1;
}
return len;
}
char* queryPlanToString(SQueryNode* pQueryNode) {
assert(pQueryNode);
char* buf = calloc(1, 4096);
int32_t len = sprintf(buf, "===== logic plan =====\n");
queryPlanToStringImpl(buf, pQueryNode, 0, len);
return buf;
}
SQueryNode* queryPlanFromString() {
return NULL;
}
SArray* createTableScanPlan(SQueryAttr* pQueryAttr) {
SArray* plan = taosArrayInit(4, sizeof(int32_t));
int32_t op = 0;
if (onlyQueryTags(pQueryAttr)) {
// op = OP_TagScan;
} else {
if (pQueryAttr->queryBlockDist) {
op = OP_TableBlockInfoScan;
} else if (pQueryAttr->tsCompQuery || pQueryAttr->pointInterpQuery || pQueryAttr->diffQuery) {
op = OP_TableSeqScan;
} else if (pQueryAttr->needReverseScan) {
op = OP_DataBlocksOptScan;
} else {
op = OP_TableScan;
}
taosArrayPush(plan, &op);
}
return plan;
}
SArray* createExecOperatorPlan(SQueryAttr* pQueryAttr) {
SArray* plan = taosArrayInit(4, sizeof(int32_t));
int32_t op = 0;
if (onlyQueryTags(pQueryAttr)) { // do nothing for tags query
op = OP_TagScan;
taosArrayPush(plan, &op);
if (pQueryAttr->distinct) {
op = OP_Distinct;
taosArrayPush(plan, &op);
}
} else if (pQueryAttr->interval.interval > 0) {
if (pQueryAttr->stableQuery) {
if (pQueryAttr->pointInterpQuery) {
op = OP_AllMultiTableTimeInterval;
} else {
op = OP_MultiTableTimeInterval;
}
taosArrayPush(plan, &op);
} else {
if (pQueryAttr->pointInterpQuery) {
op = OP_AllTimeWindow;
} else {
op = OP_TimeWindow;
}
taosArrayPush(plan, &op);
if (pQueryAttr->pExpr2 != NULL) {
op = OP_Project;
taosArrayPush(plan, &op);
}
if (pQueryAttr->fillType != TSDB_FILL_NONE) {
op = OP_Fill;
taosArrayPush(plan, &op);
}
}
} else if (pQueryAttr->groupbyColumn) {
op = OP_Groupby;
taosArrayPush(plan, &op);
if (!pQueryAttr->stableQuery && pQueryAttr->havingNum > 0) {
op = OP_Filter;
taosArrayPush(plan, &op);
}
if (pQueryAttr->pExpr2 != NULL) {
op = OP_Project;
taosArrayPush(plan, &op);
}
} else if (pQueryAttr->sw.gap > 0) {
op = OP_SessionWindow;
taosArrayPush(plan, &op);
if (pQueryAttr->pExpr2 != NULL) {
op = OP_Project;
taosArrayPush(plan, &op);
}
} else if (pQueryAttr->stateWindow) {
op = OP_StateWindow;
taosArrayPush(plan, &op);
if (pQueryAttr->pExpr2 != NULL) {
op = OP_Project;
taosArrayPush(plan, &op);
}
} else if (pQueryAttr->simpleAgg) {
if (pQueryAttr->stableQuery && !pQueryAttr->tsCompQuery && !pQueryAttr->diffQuery) {
op = OP_MultiTableAggregate;
} else {
op = OP_Aggregate;
}
taosArrayPush(plan, &op);
if (!pQueryAttr->stableQuery && pQueryAttr->havingNum > 0) {
op = OP_Filter;
taosArrayPush(plan, &op);
}
if (pQueryAttr->pExpr2 != NULL && !pQueryAttr->stableQuery) {
op = OP_Project;
taosArrayPush(plan, &op);
}
} else { // diff/add/multiply/subtract/division
if (pQueryAttr->numOfFilterCols > 0 && pQueryAttr->createFilterOperator && pQueryAttr->vgId == 0) { // todo refactor
op = OP_Filter;
taosArrayPush(plan, &op);
} else {
op = OP_Project;
taosArrayPush(plan, &op);
if (pQueryAttr->distinct) {
op = OP_Distinct;
taosArrayPush(plan, &op);
}
}
// outer query order by support
int32_t orderColId = pQueryAttr->order.orderColId;
if (pQueryAttr->vgId == 0 && orderColId != PRIMARYKEY_TIMESTAMP_COL_INDEX && orderColId != INT32_MIN) {
op = OP_Order;
taosArrayPush(plan, &op);
}
}
if (pQueryAttr->limit.limit > 0 || pQueryAttr->limit.offset > 0) {
op = OP_Limit;
taosArrayPush(plan, &op);
}
return plan;
}
SArray* createGlobalMergePlan(SQueryAttr* pQueryAttr) {
SArray* plan = taosArrayInit(4, sizeof(int32_t));
if (!pQueryAttr->stableQuery) {
return plan;
}
int32_t op = OP_MultiwayMergeSort;
taosArrayPush(plan, &op);
if (pQueryAttr->distinct) {
op = OP_Distinct;
taosArrayPush(plan, &op);
}
if (pQueryAttr->simpleAgg || (pQueryAttr->interval.interval > 0 || pQueryAttr->sw.gap > 0)) {
op = OP_GlobalAggregate;
taosArrayPush(plan, &op);
if (pQueryAttr->havingNum > 0) {
op = OP_Filter;
taosArrayPush(plan, &op);
}
if (pQueryAttr->pExpr2 != NULL) {
op = OP_Project;
taosArrayPush(plan, &op);
}
}
// fill operator
if (pQueryAttr->fillType != TSDB_FILL_NONE && pQueryAttr->interval.interval > 0) {
op = OP_Fill;
taosArrayPush(plan, &op);
}
// limit/offset operator
if (pQueryAttr->limit.limit > 0 || pQueryAttr->limit.offset > 0 ||
pQueryAttr->slimit.limit > 0 || pQueryAttr->slimit.offset > 0) {
op = OP_SLimit;
taosArrayPush(plan, &op);
}
return plan;
}

453
2.0/src/query/src/qResultbuf.c
View File

@ -1,453 +0,0 @@
#include "qResultbuf.h"
#include "hash.h"
#include "qExtbuffer.h"
#include "queryLog.h"
#include "stddef.h"
#include "taoserror.h"
#include "tcompression.h"
#define GET_DATA_PAYLOAD(_p) ((char *)(_p)->pData + POINTER_BYTES)
#define NO_IN_MEM_AVAILABLE_PAGES(_b) (listNEles((_b)->lruList) >= (_b)->inMemPages)
int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId) {
*pResultBuf = calloc(1, sizeof(SDiskbasedResultBuf));
SDiskbasedResultBuf* pResBuf = *pResultBuf;
if (pResBuf == NULL) {
return TSDB_CODE_COM_OUT_OF_MEMORY;
}
pResBuf->pageSize = pagesize;
pResBuf->numOfPages = 0; // all pages are in buffer in the first place
pResBuf->totalBufSize = 0;
pResBuf->inMemPages = inMemBufSize/pagesize; // maximum allowed pages, it is a soft limit.
pResBuf->allocateId = -1;
pResBuf->comp = true;
pResBuf->file = NULL;
pResBuf->qId = qId;
pResBuf->fileSize = 0;
// at least more than 2 pages must be in memory
assert(inMemBufSize >= pagesize * 2);
pResBuf->lruList = tdListNew(POINTER_BYTES);
// init id hash table
pResBuf->groupSet = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false);
pResBuf->assistBuf = malloc(pResBuf->pageSize + 2); // EXTRA BYTES
pResBuf->all = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false);
char path[PATH_MAX] = {0};
taosGetTmpfilePath("qbuf", path);
pResBuf->path = strdup(path);
pResBuf->emptyDummyIdList = taosArrayInit(1, sizeof(int32_t));
qDebug("QInfo:0x%"PRIx64" create resBuf for output, page size:%d, inmem buf pages:%d, file:%s", qId, pResBuf->pageSize,
pResBuf->inMemPages, pResBuf->path);
return TSDB_CODE_SUCCESS;
}
static int32_t createDiskFile(SDiskbasedResultBuf* pResultBuf) {
pResultBuf->file = fopen(pResultBuf->path, "wb+");
if (pResultBuf->file == NULL) {
qError("failed to create tmp file: %s on disk. %s", pResultBuf->path, strerror(errno));
return TAOS_SYSTEM_ERROR(errno);
}
return TSDB_CODE_SUCCESS;
}
static char* doCompressData(void* data, int32_t srcSize, int32_t *dst, SDiskbasedResultBuf* pResultBuf) { // do nothing
if (!pResultBuf->comp) {
*dst = srcSize;
return data;
}
*dst = tsCompressString(data, srcSize, 1, pResultBuf->assistBuf, srcSize, ONE_STAGE_COMP, NULL, 0);
memcpy(data, pResultBuf->assistBuf, *dst);
return data;
}
static char* doDecompressData(void* data, int32_t srcSize, int32_t *dst, SDiskbasedResultBuf* pResultBuf) { // do nothing
if (!pResultBuf->comp) {
*dst = srcSize;
return data;
}
*dst = tsDecompressString(data, srcSize, 1, pResultBuf->assistBuf, pResultBuf->pageSize, ONE_STAGE_COMP, NULL, 0);
if (*dst > 0) {
memcpy(data, pResultBuf->assistBuf, *dst);
}
return data;
}
static int32_t allocatePositionInFile(SDiskbasedResultBuf* pResultBuf, size_t size) {
if (pResultBuf->pFree == NULL) {
return pResultBuf->nextPos;
} else {
int32_t offset = -1;
size_t num = taosArrayGetSize(pResultBuf->pFree);
for(int32_t i = 0; i < num; ++i) {
SFreeListItem* pi = taosArrayGet(pResultBuf->pFree, i);
if (pi->len >= size) {
offset = pi->offset;
pi->offset += (int32_t)size;
pi->len -= (int32_t)size;
return offset;
}
}
// no available recycle space, allocate new area in file
return pResultBuf->nextPos;
}
}
static char* doFlushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) {
assert(!pg->used && pg->pData != NULL);
int32_t size = -1;
char* t = doCompressData(GET_DATA_PAYLOAD(pg), pResultBuf->pageSize, &size, pResultBuf);
// this page is flushed to disk for the first time
if (pg->info.offset == -1) {
pg->info.offset = allocatePositionInFile(pResultBuf, size);
pResultBuf->nextPos += size;
int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET);
assert(ret == 0);
ret = (int32_t) fwrite(t, 1, size, pResultBuf->file);
assert(ret == size);
if (pResultBuf->fileSize < pg->info.offset + pg->info.length) {
pResultBuf->fileSize = pg->info.offset + pg->info.length;
}
} else {
// length becomes greater, current space is not enough, allocate new place, otherwise, do nothing
if (pg->info.length < size) {
// 1. add current space to free list
taosArrayPush(pResultBuf->pFree, &pg->info);
// 2. allocate new position, and update the info
pg->info.offset = allocatePositionInFile(pResultBuf, size);
pResultBuf->nextPos += size;
}
//3. write to disk.
int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET);
if (ret != 0) { // todo handle the error case
}
ret = (int32_t)fwrite(t, size, 1, pResultBuf->file);
if (ret != size) { // todo handle the error case
}
if (pResultBuf->fileSize < pg->info.offset + pg->info.length) {
pResultBuf->fileSize = pg->info.offset + pg->info.length;
}
}
char* ret = pg->pData;
memset(ret, 0, pResultBuf->pageSize);
pg->pData = NULL;
pg->info.length = size;
pResultBuf->statis.flushBytes += pg->info.length;
return ret;
}
static char* flushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) {
int32_t ret = TSDB_CODE_SUCCESS;
assert(((int64_t) pResultBuf->numOfPages * pResultBuf->pageSize) == pResultBuf->totalBufSize && pResultBuf->numOfPages >= pResultBuf->inMemPages);
if (pResultBuf->file == NULL) {
if ((ret = createDiskFile(pResultBuf)) != TSDB_CODE_SUCCESS) {
terrno = ret;
return NULL;
}
}
return doFlushPageToDisk(pResultBuf, pg);
}
// load file block data in disk
static char* loadPageFromDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) {
int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET);
ret = (int32_t)fread(GET_DATA_PAYLOAD(pg), 1, pg->info.length, pResultBuf->file);
if (ret != pg->info.length) {
terrno = errno;
return NULL;
}
pResultBuf->statis.loadBytes += pg->info.length;
int32_t fullSize = 0;
doDecompressData(GET_DATA_PAYLOAD(pg), pg->info.length, &fullSize, pResultBuf);
return (char*)GET_DATA_PAYLOAD(pg);
}
static SIDList addNewGroup(SDiskbasedResultBuf* pResultBuf, int32_t groupId) {
assert(taosHashGet(pResultBuf->groupSet, (const char*) &groupId, sizeof(int32_t)) == NULL);
SArray* pa = taosArrayInit(1, POINTER_BYTES);
int32_t ret = taosHashPut(pResultBuf->groupSet, (const char*)&groupId, sizeof(int32_t), &pa, POINTER_BYTES);
assert(ret == 0);
return pa;
}
static SPageInfo* registerPage(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t pageId) {
SIDList list = NULL;
char** p = taosHashGet(pResultBuf->groupSet, (const char*)&groupId, sizeof(int32_t));
if (p == NULL) { // it is a new group id
list = addNewGroup(pResultBuf, groupId);
} else {
list = (SIDList) (*p);
}
pResultBuf->numOfPages += 1;
SPageInfo* ppi = malloc(sizeof(SPageInfo));//{ .info = PAGE_INFO_INITIALIZER, .pageId = pageId, .pn = NULL};
ppi->pageId = pageId;
ppi->pData = NULL;
ppi->info = PAGE_INFO_INITIALIZER;
ppi->used = true;
ppi->pn = NULL;
return *(SPageInfo**) taosArrayPush(list, &ppi);
}
static SListNode* getEldestUnrefedPage(SDiskbasedResultBuf* pResultBuf) {
SListIter iter = {0};
tdListInitIter(pResultBuf->lruList, &iter, TD_LIST_BACKWARD);
SListNode* pn = NULL;
while((pn = tdListNext(&iter)) != NULL) {
assert(pn != NULL);
SPageInfo* pageInfo = *(SPageInfo**) pn->data;
assert(pageInfo->pageId >= 0 && pageInfo->pn == pn);
if (!pageInfo->used) {
break;
}
}
return pn;
}
static char* evicOneDataPage(SDiskbasedResultBuf* pResultBuf) {
char* bufPage = NULL;
SListNode* pn = getEldestUnrefedPage(pResultBuf);
// all pages are referenced by user, try to allocate new space
if (pn == NULL) {
int32_t prev = pResultBuf->inMemPages;
// increase by 50% of previous mem pages
pResultBuf->inMemPages = (int32_t)(pResultBuf->inMemPages * 1.5f);
qWarn("%p in memory buf page not sufficient, expand from %d to %d, page size:%d", pResultBuf, prev,
pResultBuf->inMemPages, pResultBuf->pageSize);
} else {
pResultBuf->statis.flushPages += 1;
tdListPopNode(pResultBuf->lruList, pn);
SPageInfo* d = *(SPageInfo**) pn->data;
assert(d->pn == pn);
d->pn = NULL;
tfree(pn);
bufPage = flushPageToDisk(pResultBuf, d);
}
return bufPage;
}
static void lruListPushFront(SList *pList, SPageInfo* pi) {
tdListPrepend(pList, &pi);
SListNode* front = tdListGetHead(pList);
pi->pn = front;
}
static void lruListMoveToFront(SList *pList, SPageInfo* pi) {
tdListPopNode(pList, pi->pn);
tdListPrependNode(pList, pi->pn);
}
static FORCE_INLINE size_t getAllocPageSize(int32_t pageSize) {
return pageSize + POINTER_BYTES + 2 + sizeof(tFilePage);
}
tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId) {
pResultBuf->statis.getPages += 1;
char* availablePage = NULL;
if (NO_IN_MEM_AVAILABLE_PAGES(pResultBuf)) {
availablePage = evicOneDataPage(pResultBuf);
}
// register new id in this group
*pageId = (++pResultBuf->allocateId);
// register page id info
SPageInfo* pi = registerPage(pResultBuf, groupId, *pageId);
// add to LRU list
assert(listNEles(pResultBuf->lruList) < pResultBuf->inMemPages && pResultBuf->inMemPages > 0);
lruListPushFront(pResultBuf->lruList, pi);
// add to hash map
taosHashPut(pResultBuf->all, pageId, sizeof(int32_t), &pi, POINTER_BYTES);
// allocate buf
if (availablePage == NULL) {
pi->pData = calloc(1, getAllocPageSize(pResultBuf->pageSize)); // add extract bytes in case of zipped buffer increased.
} else {
pi->pData = availablePage;
}
pResultBuf->totalBufSize += pResultBuf->pageSize;
((void**)pi->pData)[0] = pi;
pi->used = true;
return (void *)(GET_DATA_PAYLOAD(pi));
}
tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id) {
assert(pResultBuf != NULL && id >= 0);
pResultBuf->statis.getPages += 1;
SPageInfo** pi = taosHashGet(pResultBuf->all, &id, sizeof(int32_t));
assert(pi != NULL && *pi != NULL);
if ((*pi)->pData != NULL) { // it is in memory
// no need to update the LRU list if only one page exists
if (pResultBuf->numOfPages == 1) {
(*pi)->used = true;
return (void *)(GET_DATA_PAYLOAD(*pi));
}
SPageInfo** pInfo = (SPageInfo**) ((*pi)->pn->data);
assert(*pInfo == *pi);
lruListMoveToFront(pResultBuf->lruList, (*pi));
(*pi)->used = true;
return (void *)(GET_DATA_PAYLOAD(*pi));
} else { // not in memory
assert((*pi)->pData == NULL && (*pi)->pn == NULL && (*pi)->info.length >= 0 && (*pi)->info.offset >= 0);
char* availablePage = NULL;
if (NO_IN_MEM_AVAILABLE_PAGES(pResultBuf)) {
availablePage = evicOneDataPage(pResultBuf);
}
if (availablePage == NULL) {
(*pi)->pData = calloc(1, getAllocPageSize(pResultBuf->pageSize));
} else {
(*pi)->pData = availablePage;
}
((void**)((*pi)->pData))[0] = (*pi);
lruListPushFront(pResultBuf->lruList, *pi);
(*pi)->used = true;
loadPageFromDisk(pResultBuf, *pi);
return (void *)(GET_DATA_PAYLOAD(*pi));
}
}
void releaseResBufPage(SDiskbasedResultBuf* pResultBuf, void* page) {
assert(pResultBuf != NULL && page != NULL);
char* p = (char*) page - POINTER_BYTES;
SPageInfo* ppi = ((SPageInfo**) p)[0];
releaseResBufPageInfo(pResultBuf, ppi);
}
void releaseResBufPageInfo(SDiskbasedResultBuf* pResultBuf, SPageInfo* pi) {
assert(pi->pData != NULL && pi->used);
pi->used = false;
pResultBuf->statis.releasePages += 1;
}
size_t getNumOfResultBufGroupId(const SDiskbasedResultBuf* pResultBuf) { return taosHashGetSize(pResultBuf->groupSet); }
size_t getResBufSize(const SDiskbasedResultBuf* pResultBuf) { return (size_t)pResultBuf->totalBufSize; }
SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId) {
assert(pResultBuf != NULL);
char** p = taosHashGet(pResultBuf->groupSet, (const char*)&groupId, sizeof(int32_t));
if (p == NULL) { // it is a new group id
return pResultBuf->emptyDummyIdList;
} else {
return (SArray*) (*p);
}
}
void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) {
if (pResultBuf == NULL) {
return;
}
if (pResultBuf->file != NULL) {
qDebug("QInfo:0x%"PRIx64" res output buffer closed, total:%.2f Kb, inmem size:%.2f Kb, file size:%.2f Kb",
pResultBuf->qId, pResultBuf->totalBufSize/1024.0, listNEles(pResultBuf->lruList) * pResultBuf->pageSize / 1024.0,
pResultBuf->fileSize/1024.0);
fclose(pResultBuf->file);
} else {
qDebug("QInfo:0x%"PRIx64" res output buffer closed, total:%.2f Kb, no file created", pResultBuf->qId,
pResultBuf->totalBufSize/1024.0);
}
unlink(pResultBuf->path);
tfree(pResultBuf->path);
SArray** p = taosHashIterate(pResultBuf->groupSet, NULL);
while(p) {
size_t n = taosArrayGetSize(*p);
for(int32_t i = 0; i < n; ++i) {
SPageInfo* pi = taosArrayGetP(*p, i);
tfree(pi->pData);
tfree(pi);
}
taosArrayDestroy(*p);
p = taosHashIterate(pResultBuf->groupSet, p);
}
tdListFree(pResultBuf->lruList);
taosArrayDestroy(pResultBuf->emptyDummyIdList);
taosHashCleanup(pResultBuf->groupSet);
taosHashCleanup(pResultBuf->all);
tfree(pResultBuf->assistBuf);
tfree(pResultBuf);
}
SPageInfo* getLastPageInfo(SIDList pList) {
size_t size = taosArrayGetSize(pList);
return (SPageInfo*) taosArrayGetP(pList, size - 1);
}

1209
2.0/src/query/src/qSqlParser.c
View File

File diff suppressed because it is too large Load Diff

107
2.0/src/query/src/qTableMeta.c
View File

@ -1,107 +0,0 @@
#include "os.h"
#include "tmsg.h"
#include "qTableMeta.h"
#include "ttokendef.h"
#include "taosdef.h"
#include "tutil.h"
int32_t tscGetNumOfTags(const STableMeta* pTableMeta) {
assert(pTableMeta != NULL);
STableComInfo tinfo = tscGetTableInfo(pTableMeta);
if (pTableMeta->tableType == TSDB_NORMAL_TABLE) {
assert(tinfo.numOfTags == 0);
return 0;
}
if (pTableMeta->tableType == TSDB_SUPER_TABLE || pTableMeta->tableType == TSDB_CHILD_TABLE) {
return tinfo.numOfTags;
}
assert(tinfo.numOfTags == 0);
return 0;
}
int32_t tscGetNumOfColumns(const STableMeta* pTableMeta) {
assert(pTableMeta != NULL);
// table created according to super table, use data from super table
STableComInfo tinfo = tscGetTableInfo(pTableMeta);
return tinfo.numOfColumns;
}
SSchema *tscGetTableSchema(const STableMeta *pTableMeta) {
assert(pTableMeta != NULL);
return (SSchema*) pTableMeta->schema;
}
SSchema* tscGetTableTagSchema(const STableMeta* pTableMeta) {
assert(pTableMeta != NULL && (pTableMeta->tableType == TSDB_SUPER_TABLE || pTableMeta->tableType == TSDB_CHILD_TABLE));
STableComInfo tinfo = tscGetTableInfo(pTableMeta);
assert(tinfo.numOfTags > 0);
return tscGetTableColumnSchema(pTableMeta, tinfo.numOfColumns);
}
STableComInfo tscGetTableInfo(const STableMeta* pTableMeta) {
assert(pTableMeta != NULL);
return pTableMeta->tableInfo;
}
SSchema* tscGetTableColumnSchema(const STableMeta* pTableMeta, int32_t colIndex) {
assert(pTableMeta != NULL);
SSchema* pSchema = (SSchema*) pTableMeta->schema;
return &pSchema[colIndex];
}
// TODO for large number of columns, employ the binary search method
SSchema* tscGetColumnSchemaById(STableMeta* pTableMeta, int16_t colId) {
STableComInfo tinfo = tscGetTableInfo(pTableMeta);
for(int32_t i = 0; i < tinfo.numOfColumns + tinfo.numOfTags; ++i) {
if (pTableMeta->schema[i].colId == colId) {
return &pTableMeta->schema[i];
}
}
return NULL;
}
STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg) {
assert(pTableMetaMsg != NULL && pTableMetaMsg->numOfColumns >= 2);
int32_t schemaSize = (pTableMetaMsg->numOfColumns + pTableMetaMsg->numOfTags) * sizeof(SSchema);
STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + schemaSize);
pTableMeta->tableType = pTableMetaMsg->tableType;
pTableMeta->vgId = pTableMetaMsg->vgroup.vgId;
pTableMeta->suid = pTableMetaMsg->suid;
pTableMeta->tableInfo = (STableComInfo) {
.numOfTags = pTableMetaMsg->numOfTags,
.precision = pTableMetaMsg->precision,
.numOfColumns = pTableMetaMsg->numOfColumns,
};
pTableMeta->id.tid = pTableMetaMsg->tid;
pTableMeta->id.uid = pTableMetaMsg->uid;
pTableMeta->sversion = pTableMetaMsg->sversion;
pTableMeta->tversion = pTableMetaMsg->tversion;
tstrncpy(pTableMeta->sTableName, pTableMetaMsg->sTableName, TSDB_TABLE_FNAME_LEN);
memcpy(pTableMeta->schema, pTableMetaMsg->schema, schemaSize);
int32_t numOfTotalCols = pTableMeta->tableInfo.numOfColumns;
for(int32_t i = 0; i < numOfTotalCols; ++i) {
pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes;
}
return pTableMeta;
}

1109
2.0/src/query/src/qTsbuf.c
View File

File diff suppressed because it is too large Load Diff

769
2.0/src/query/src/queryMain.c
View File

@ -1,769 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "tmsg.h"
#include "tcache.h"
#include "tglobal.h"
#include "exception.h"
#include "hash.h"
#include "texpr.h"
#include "qExecutor.h"
#include "qUtil.h"
#include "query.h"
#include "queryLog.h"
#include "tlosertree.h"
#include "ttype.h"
typedef struct SQueryMgmt {
pthread_mutex_t lock;
SCacheObj *qinfoPool; // query handle pool
int32_t vgId;
bool closed;
} SQueryMgmt;
static void queryMgmtKillQueryFn(void* handle, void* param1) {
void** fp = (void**)handle;
qKillQuery(*fp);
}
static void freeqinfoFn(void *qhandle) {
void** handle = qhandle;
if (handle == NULL || *handle == NULL) {
return;
}
qKillQuery(*handle);
qDestroyQueryInfo(*handle);
}
void freeParam(SQueryParam *param) {
tfree(param->sql);
tfree(param->tagCond);
tfree(param->tbnameCond);
tfree(param->pTableIdList);
taosArrayDestroy(param->pOperator);
tfree(param->pExprs);
tfree(param->pSecExprs);
tfree(param->pExpr);
tfree(param->pSecExpr);
tfree(param->pGroupColIndex);
tfree(param->pTagColumnInfo);
tfree(param->pGroupbyExpr);
tfree(param->prevResult);
}
int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qinfo_t* pQInfo, uint64_t qId) {
assert(pQueryMsg != NULL && tsdb != NULL);
int32_t code = TSDB_CODE_SUCCESS;
SQueryParam param = {0};
code = convertQueryMsg(pQueryMsg, &param);
if (code != TSDB_CODE_SUCCESS) {
goto _over;
}
if (pQueryMsg->numOfTables <= 0) {
qError("Invalid number of tables to query, numOfTables:%d", pQueryMsg->numOfTables);
code = TSDB_CODE_QRY_INVALID_MSG;
goto _over;
}
if (param.pTableIdList == NULL || taosArrayGetSize(param.pTableIdList) == 0) {
qError("qmsg:%p, SQueryTableMsg wrong format", pQueryMsg);
code = TSDB_CODE_QRY_INVALID_MSG;
goto _over;
}
SQueriedTableInfo info = { .numOfTags = pQueryMsg->numOfTags, .numOfCols = pQueryMsg->numOfCols, .colList = pQueryMsg->tableCols};
if ((code = createQueryFunc(&info, pQueryMsg->numOfOutput, &param.pExprs, param.pExpr, param.pTagColumnInfo,
pQueryMsg->queryType, pQueryMsg, param.pUdfInfo)) != TSDB_CODE_SUCCESS) {
goto _over;
}
if (param.pSecExpr != NULL) {
if ((code = createIndirectQueryFuncExprFromMsg(pQueryMsg, pQueryMsg->secondStageOutput, &param.pSecExprs, param.pSecExpr, param.pExprs, param.pUdfInfo)) != TSDB_CODE_SUCCESS) {
goto _over;
}
}
if (param.colCond != NULL) {
if ((code = createQueryFilter(param.colCond, pQueryMsg->colCondLen, &param.pFilters)) != TSDB_CODE_SUCCESS) {
goto _over;
}
}
param.pGroupbyExpr = createGroupbyExprFromMsg(pQueryMsg, param.pGroupColIndex, &code);
if ((param.pGroupbyExpr == NULL && pQueryMsg->numOfGroupCols != 0) || code != TSDB_CODE_SUCCESS) {
goto _over;
}
bool isSTableQuery = false;
STableGroupInfo tableGroupInfo = {0};
int64_t st = taosGetTimestampUs();
if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_TABLE_QUERY)) {
STableIdInfo *id = taosArrayGet(param.pTableIdList, 0);
qDebug("qmsg:%p query normal table, uid:%"PRId64", tid:%d", pQueryMsg, id->uid, id->tid);
if ((code = tsdbGetOneTableGroup(tsdb, id->uid, pQueryMsg->window.skey, &tableGroupInfo)) != TSDB_CODE_SUCCESS) {
goto _over;
}
} else if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY|TSDB_QUERY_TYPE_STABLE_QUERY)) {
isSTableQuery = true;
// also note there's possibility that only one table in the super table
if (!TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY)) {
STableIdInfo *id = taosArrayGet(param.pTableIdList, 0);
// group by normal column, do not pass the group by condition to tsdb to group table into different group
int32_t numOfGroupByCols = pQueryMsg->numOfGroupCols;
if (pQueryMsg->numOfGroupCols == 1 && !TSDB_COL_IS_TAG(param.pGroupColIndex->flag)) {
numOfGroupByCols = 0;
}
qDebug("qmsg:%p query stable, uid:%"PRIu64", tid:%d", pQueryMsg, id->uid, id->tid);
code = tsdbQuerySTableByTagCond(tsdb, id->uid, pQueryMsg->window.skey, param.tagCond, pQueryMsg->tagCondLen,
pQueryMsg->tagNameRelType, param.tbnameCond, &tableGroupInfo, param.pGroupColIndex, numOfGroupByCols);
if (code != TSDB_CODE_SUCCESS) {
qError("qmsg:%p failed to query stable, reason: %s", pQueryMsg, tstrerror(code));
goto _over;
}
} else {
code = tsdbGetTableGroupFromIdList(tsdb, param.pTableIdList, &tableGroupInfo);
if (code != TSDB_CODE_SUCCESS) {
goto _over;
}
qDebug("qmsg:%p query on %u tables in one group from client", pQueryMsg, tableGroupInfo.numOfTables);
}
int64_t el = taosGetTimestampUs() - st;
qDebug("qmsg:%p tag filter completed, numOfTables:%u, elapsed time:%"PRId64"us", pQueryMsg, tableGroupInfo.numOfTables, el);
} else {
assert(0);
}
code = checkForQueryBuf(tableGroupInfo.numOfTables);
if (code != TSDB_CODE_SUCCESS) { // not enough query buffer, abort
goto _over;
}
assert(pQueryMsg->stableQuery == isSTableQuery);
(*pQInfo) = createQInfoImpl(pQueryMsg, param.pGroupbyExpr, param.pExprs, param.pSecExprs, &tableGroupInfo,
param.pTagColumnInfo, param.pFilters, vgId, param.sql, qId, param.pUdfInfo);
param.sql = NULL;
param.pExprs = NULL;
param.pSecExprs = NULL;
param.pGroupbyExpr = NULL;
param.pTagColumnInfo = NULL;
param.pFilters = NULL;
if ((*pQInfo) == NULL) {
code = TSDB_CODE_QRY_OUT_OF_MEMORY;
goto _over;
}
param.pUdfInfo = NULL;
code = initQInfo(&pQueryMsg->tsBuf, tsdb, NULL, *pQInfo, &param, (char*)pQueryMsg, pQueryMsg->prevResultLen, NULL);
_over:
if (param.pGroupbyExpr != NULL) {
taosArrayDestroy(param.pGroupbyExpr->columnInfo);
}
tfree(param.colCond);
destroyUdfInfo(param.pUdfInfo);
taosArrayDestroy(param.pTableIdList);
param.pTableIdList = NULL;
freeParam(&param);
for (int32_t i = 0; i < pQueryMsg->numOfCols; i++) {
SColumnInfo* column = pQueryMsg->tableCols + i;
freeColumnFilterInfo(column->flist.filterInfo, column->flist.numOfFilters);
}
filterFreeInfo(param.pFilters);
//pQInfo already freed in initQInfo, but *pQInfo may not pointer to null;
if (code != TSDB_CODE_SUCCESS) {
*pQInfo = NULL;
}
// if failed to add ref for all tables in this query, abort current query
return code;
}
#ifdef TEST_IMPL
// wait moment
int waitMoment(SQInfo* pQInfo){
if(pQInfo->sql) {
int ms = 0;
char* pcnt = strstr(pQInfo->sql, " count(*)");
if(pcnt) return 0;
char* pos = strstr(pQInfo->sql, " t_");
if(pos){
pos += 3;
ms = atoi(pos);
while(*pos >= '0' && *pos <= '9'){
pos ++;
}
char unit_char = *pos;
if(unit_char == 'h'){
ms *= 3600*1000;
} else if(unit_char == 'm'){
ms *= 60*1000;
} else if(unit_char == 's'){
ms *= 1000;
}
}
if(ms == 0) return 0;
printf("test wait sleep %dms. sql=%s ...\n", ms, pQInfo->sql);
if(ms < 1000) {
taosMsleep(ms);
} else {
int used_ms = 0;
while(used_ms < ms) {
taosMsleep(1000);
used_ms += 1000;
if(isQueryKilled(pQInfo)){
printf("test check query is canceled, sleep break.%s\n", pQInfo->sql);
break;
}
}
}
}
return 1;
}
#endif
bool qTableQuery(qinfo_t qinfo, uint64_t *qId) {
SQInfo *pQInfo = (SQInfo *)qinfo;
assert(pQInfo && pQInfo->signature == pQInfo);
int64_t threadId = taosGetSelfPthreadId();
int64_t curOwner = 0;
if ((curOwner = atomic_val_compare_exchange_64(&pQInfo->owner, 0, threadId)) != 0) {
qError("QInfo:0x%"PRIx64"-%p qhandle is now executed by thread:%p", pQInfo->qId, pQInfo, (void*) curOwner);
pQInfo->code = TSDB_CODE_QRY_IN_EXEC;
return false;
}
*qId = pQInfo->qId;
if(pQInfo->startExecTs == 0)
pQInfo->startExecTs = taosGetTimestampMs();
if (isQueryKilled(pQInfo)) {
qDebug("QInfo:0x%"PRIx64" it is already killed, abort", pQInfo->qId);
return doBuildResCheck(pQInfo);
}
SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv;
if (pRuntimeEnv->tableqinfoGroupInfo.numOfTables == 0) {
qDebug("QInfo:0x%"PRIx64" no table exists for query, abort", pQInfo->qId);
setQueryStatus(pRuntimeEnv, QUERY_COMPLETED);
return doBuildResCheck(pQInfo);
}
// error occurs, record the error code and return to client
int32_t ret = setjmp(pQInfo->runtimeEnv.env);
if (ret != TSDB_CODE_SUCCESS) {
publishQueryAbortEvent(pQInfo, ret);
pQInfo->code = ret;
qDebug("QInfo:0x%"PRIx64" query abort due to error/cancel occurs, code:%s", pQInfo->qId, tstrerror(pQInfo->code));
return doBuildResCheck(pQInfo);
}
qDebug("QInfo:0x%"PRIx64" query task is launched", pQInfo->qId);
bool newgroup = false;
publishOperatorProfEvent(pRuntimeEnv->proot, QUERY_PROF_BEFORE_OPERATOR_EXEC);
int64_t st = taosGetTimestampUs();
pRuntimeEnv->outputBuf = pRuntimeEnv->proot->exec(pRuntimeEnv->proot, &newgroup);
pQInfo->summary.elapsedTime += (taosGetTimestampUs() - st);
#ifdef TEST_IMPL
waitMoment(pQInfo);
#endif
publishOperatorProfEvent(pRuntimeEnv->proot, QUERY_PROF_AFTER_OPERATOR_EXEC);
pRuntimeEnv->resultInfo.total += GET_NUM_OF_RESULTS(pRuntimeEnv);
if (isQueryKilled(pQInfo)) {
qDebug("QInfo:0x%"PRIx64" query is killed", pQInfo->qId);
} else if (GET_NUM_OF_RESULTS(pRuntimeEnv) == 0) {
qDebug("QInfo:0x%"PRIx64" over, %u tables queried, total %"PRId64" rows returned", pQInfo->qId, pRuntimeEnv->tableqinfoGroupInfo.numOfTables,
pRuntimeEnv->resultInfo.total);
} else {
qDebug("QInfo:0x%"PRIx64" query paused, %d rows returned, total:%" PRId64 " rows", pQInfo->qId,
GET_NUM_OF_RESULTS(pRuntimeEnv), pRuntimeEnv->resultInfo.total);
}
return doBuildResCheck(pQInfo);
}
int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContext) {
SQInfo *pQInfo = (SQInfo *)qinfo;
if (pQInfo == NULL || !isValidQInfo(pQInfo)) {
qError("QInfo invalid qhandle");
return TSDB_CODE_QRY_INVALID_QHANDLE;
}
*buildRes = false;
if (IS_QUERY_KILLED(pQInfo)) {
qDebug("QInfo:0x%"PRIx64" query is killed, code:0x%08x", pQInfo->qId, pQInfo->code);
return pQInfo->code;
}
int32_t code = TSDB_CODE_SUCCESS;
if (tsRetrieveBlockingModel) {
pQInfo->rspContext = pRspContext;
tsem_wait(&pQInfo->ready);
*buildRes = true;
code = pQInfo->code;
} else {
SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv;
SQueryAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr;
pthread_mutex_lock(&pQInfo->lock);
assert(pQInfo->rspContext == NULL);
if (pQInfo->dataReady == QUERY_RESULT_READY) {
*buildRes = true;
qDebug("QInfo:0x%"PRIx64" retrieve result info, rowsize:%d, rows:%d, code:%s", pQInfo->qId, pQueryAttr->resultRowSize,
GET_NUM_OF_RESULTS(pRuntimeEnv), tstrerror(pQInfo->code));
} else {
*buildRes = false;
qDebug("QInfo:0x%"PRIx64" retrieve req set query return result after paused", pQInfo->qId);
pQInfo->rspContext = pRspContext;
assert(pQInfo->rspContext != NULL);
}
code = pQInfo->code;
pthread_mutex_unlock(&pQInfo->lock);
}
return code;
}
int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *contLen, bool* continueExec) {
SQInfo *pQInfo = (SQInfo *)qinfo;
int32_t compLen = 0;
if (pQInfo == NULL || !isValidQInfo(pQInfo)) {
return TSDB_CODE_QRY_INVALID_QHANDLE;
}
SQueryAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr;
SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv;
int32_t s = GET_NUM_OF_RESULTS(pRuntimeEnv);
size_t size = pQueryAttr->resultRowSize * s;
size += sizeof(int32_t);
size += sizeof(STableIdInfo) * taosHashGetSize(pRuntimeEnv->pTableRetrieveTsMap);
*contLen = (int32_t)(size + sizeof(SRetrieveTableRsp));
// current solution only avoid crash, but cannot return error code to client
*pRsp = (SRetrieveTableRsp *)rpcMallocCont(*contLen);
if (*pRsp == NULL) {
return TSDB_CODE_QRY_OUT_OF_MEMORY;
}
(*pRsp)->numOfRows = htonl((int32_t)s);
if (pQInfo->code == TSDB_CODE_SUCCESS) {
(*pRsp)->offset = htobe64(pQInfo->runtimeEnv.currentOffset);
(*pRsp)->useconds = htobe64(pQInfo->summary.elapsedTime);
} else {
(*pRsp)->offset = 0;
(*pRsp)->useconds = htobe64(pQInfo->summary.elapsedTime);
}
(*pRsp)->precision = htons(pQueryAttr->precision);
(*pRsp)->compressed = (int8_t)((tsCompressColData != -1) && checkNeedToCompressQueryCol(pQInfo));
if (GET_NUM_OF_RESULTS(&(pQInfo->runtimeEnv)) > 0 && pQInfo->code == TSDB_CODE_SUCCESS) {
doDumpQueryResult(pQInfo, (*pRsp)->data, (*pRsp)->compressed, &compLen);
} else {
setQueryStatus(pRuntimeEnv, QUERY_OVER);
}
if ((*pRsp)->compressed && compLen != 0) {
int32_t numOfCols = pQueryAttr->pExpr2 ? pQueryAttr->numOfExpr2 : pQueryAttr->numOfOutput;
int32_t origSize = pQueryAttr->resultRowSize * s;
int32_t compSize = compLen + numOfCols * sizeof(int32_t);
*contLen = *contLen - origSize + compSize;
*pRsp = (SRetrieveTableRsp *)rpcReallocCont(*pRsp, *contLen);
qDebug("QInfo:0x%"PRIx64" compress col data, uncompressed size:%d, compressed size:%d, ratio:%.2f",
pQInfo->qId, origSize, compSize, (float)origSize / (float)compSize);
}
(*pRsp)->compLen = htonl(compLen);
pQInfo->rspContext = NULL;
pQInfo->dataReady = QUERY_RESULT_NOT_READY;
if (IS_QUERY_KILLED(pQInfo) || Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_OVER)) {
// here current thread hold the refcount, so it is safe to free tsdbQueryHandle.
*continueExec = false;
(*pRsp)->completed = 1; // notify no more result to client
qDebug("QInfo:0x%"PRIx64" no more results to retrieve", pQInfo->qId);
} else {
*continueExec = true;
qDebug("QInfo:0x%"PRIx64" has more results to retrieve", pQInfo->qId);
}
// the memory should be freed if the code of pQInfo is not TSDB_CODE_SUCCESS
if (pQInfo->code != TSDB_CODE_SUCCESS) {
rpcFreeCont(*pRsp);
*pRsp = NULL;
}
return pQInfo->code;
}
void* qGetResultRetrieveMsg(qinfo_t qinfo) {
SQInfo* pQInfo = (SQInfo*) qinfo;
assert(pQInfo != NULL);
return pQInfo->rspContext;
}
int32_t qKillQuery(qinfo_t qinfo) {
SQInfo *pQInfo = (SQInfo *)qinfo;
if (pQInfo == NULL || !isValidQInfo(pQInfo)) {
return TSDB_CODE_QRY_INVALID_QHANDLE;
}
qDebug("QInfo:0x%"PRIx64" query killed", pQInfo->qId);
setTaskKilled(pQInfo);
// Wait for the query executing thread being stopped/
// Once the query is stopped, the owner of qHandle will be cleared immediately.
while (pQInfo->owner != 0) {
taosMsleep(100);
}
return TSDB_CODE_SUCCESS;
}
int32_t qQueryCompleted(qinfo_t qinfo) {
SQInfo *pQInfo = (SQInfo *)qinfo;
if (pQInfo == NULL || !isValidQInfo(pQInfo)) {
return TSDB_CODE_QRY_INVALID_QHANDLE;
}
return isQueryKilled(pQInfo) || Q_STATUS_EQUAL(pQInfo->runtimeEnv.status, QUERY_OVER);
}
void qDestroyQueryInfo(qinfo_t qHandle) {
SQInfo* pQInfo = (SQInfo*) qHandle;
if (!isValidQInfo(pQInfo)) {
return;
}
qDebug("QInfo:0x%"PRIx64" query completed", pQInfo->qId);
queryCostStatis(pQInfo); // print the query cost summary
freeQInfo(pQInfo);
}
void* qOpenQueryMgmt(int32_t vgId) {
const int32_t refreshHandleInterval = 30; // every 30 seconds, refresh handle pool
char cacheName[128] = {0};
sprintf(cacheName, "qhandle_%d", vgId);
SQueryMgmt* pQueryMgmt = calloc(1, sizeof(SQueryMgmt));
if (pQueryMgmt == NULL) {
terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
return NULL;
}
pQueryMgmt->qinfoPool = taosCacheInit(TSDB_CACHE_PTR_KEY, refreshHandleInterval, true, freeqinfoFn, cacheName);
pQueryMgmt->closed = false;
pQueryMgmt->vgId = vgId;
pthread_mutex_init(&pQueryMgmt->lock, NULL);
qDebug("vgId:%d, open querymgmt success", vgId);
return pQueryMgmt;
}
void qQueryMgmtNotifyClosed(void* pQMgmt) {
if (pQMgmt == NULL) {
return;
}
SQueryMgmt* pQueryMgmt = pQMgmt;
qInfo("vgId:%d, set querymgmt closed, wait for all queries cancelled", pQueryMgmt->vgId);
pthread_mutex_lock(&pQueryMgmt->lock);
pQueryMgmt->closed = true;
pthread_mutex_unlock(&pQueryMgmt->lock);
taosCacheRefresh(pQueryMgmt->qinfoPool, queryMgmtKillQueryFn, NULL);
}
void qQueryMgmtReOpen(void *pQMgmt) {
if (pQMgmt == NULL) {
return;
}
SQueryMgmt *pQueryMgmt = pQMgmt;
qInfo("vgId:%d, set querymgmt reopen", pQueryMgmt->vgId);
pthread_mutex_lock(&pQueryMgmt->lock);
pQueryMgmt->closed = false;
pthread_mutex_unlock(&pQueryMgmt->lock);
}
void qCleanupQueryMgmt(void* pQMgmt) {
if (pQMgmt == NULL) {
return;
}
SQueryMgmt* pQueryMgmt = pQMgmt;
int32_t vgId = pQueryMgmt->vgId;
assert(pQueryMgmt->closed);
SCacheObj* pqinfoPool = pQueryMgmt->qinfoPool;
pQueryMgmt->qinfoPool = NULL;
taosCacheCleanup(pqinfoPool);
pthread_mutex_destroy(&pQueryMgmt->lock);
tfree(pQueryMgmt);
qDebug("vgId:%d, queryMgmt cleanup completed", vgId);
}
void** qRegisterQInfo(void* pMgmt, uint64_t qId, void *qInfo) {
if (pMgmt == NULL) {
terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;
return NULL;
}
SQueryMgmt *pQueryMgmt = pMgmt;
if (pQueryMgmt->qinfoPool == NULL) {
qError("QInfo:0x%"PRIx64"-%p failed to add qhandle into qMgmt, since qMgmt is closed", qId, (void*)qInfo);
terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;
return NULL;
}
pthread_mutex_lock(&pQueryMgmt->lock);
if (pQueryMgmt->closed) {
pthread_mutex_unlock(&pQueryMgmt->lock);
qError("QInfo:0x%"PRIx64"-%p failed to add qhandle into cache, since qMgmt is colsing", qId, (void*)qInfo);
terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;
return NULL;
} else {
void** handle = taosCachePut(pQueryMgmt->qinfoPool, &qId, sizeof(qId), &qInfo, sizeof(TSDB_CACHE_PTR_TYPE),
(getMaximumIdleDurationSec()*1000));
pthread_mutex_unlock(&pQueryMgmt->lock);
return handle;
}
}
void** qAcquireQInfo(void* pMgmt, uint64_t _key) {
SQueryMgmt *pQueryMgmt = pMgmt;
if (pQueryMgmt->closed) {
terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;
return NULL;
}
if (pQueryMgmt->qinfoPool == NULL) {
terrno = TSDB_CODE_QRY_INVALID_QHANDLE;
return NULL;
}
void** handle = taosCacheAcquireByKey(pQueryMgmt->qinfoPool, &_key, sizeof(_key));
if (handle == NULL || *handle == NULL) {
terrno = TSDB_CODE_QRY_INVALID_QHANDLE;
return NULL;
} else {
return handle;
}
}
void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool freeHandle) {
SQueryMgmt *pQueryMgmt = pMgmt;
if (pQueryMgmt->qinfoPool == NULL) {
return NULL;
}
taosCacheRelease(pQueryMgmt->qinfoPool, pQInfo, freeHandle);
return 0;
}
//kill by qid
int32_t qKillQueryByQId(void* pMgmt, int64_t qId, int32_t waitMs, int32_t waitCount) {
int32_t error = TSDB_CODE_SUCCESS;
void** handle = qAcquireQInfo(pMgmt, qId);
if(handle == NULL) return terrno;
SQInfo* pQInfo = (SQInfo*)(*handle);
if (pQInfo == NULL || !isValidQInfo(pQInfo)) {
return TSDB_CODE_QRY_INVALID_QHANDLE;
}
qWarn("QId:0x%"PRIx64" be killed(no memory commit).", pQInfo->qId);
setTaskKilled(pQInfo);
// wait query stop
int32_t loop = 0;
while (pQInfo->owner != 0) {
taosMsleep(waitMs);
if(loop++ > waitCount){
error = TSDB_CODE_FAILED;
break;
}
}
qReleaseQInfo(pMgmt, (void **)&handle, true);
return error;
}
// local struct
typedef struct {
int64_t qId;
int64_t startExecTs;
} SLongQuery;
// callbark for sort compare
static int compareLongQuery(const void* p1, const void* p2) {
// sort desc
SLongQuery* plq1 = *(SLongQuery**)p1;
SLongQuery* plq2 = *(SLongQuery**)p2;
if(plq1->startExecTs == plq2->startExecTs) {
return 0;
} else if(plq1->startExecTs > plq2->startExecTs) {
return 1;
} else {
return -1;
}
}
// callback for taosCacheRefresh
static void cbFoundItem(void* handle, void* param1) {
SQInfo * qInfo = *(SQInfo**) handle;
if(qInfo == NULL) return ;
SArray* qids = (SArray*) param1;
if(qids == NULL) return ;
bool usedMem = true;
bool usedIMem = true;
SMemTable* mem = qInfo->query.memRef.snapshot.omem;
SMemTable* imem = qInfo->query.memRef.snapshot.imem;
if(mem == NULL || T_REF_VAL_GET(mem) == 0)
usedMem = false;
if(imem == NULL || T_REF_VAL_GET(mem) == 0)
usedIMem = false ;
if(!usedMem && !usedIMem)
return ;
// push to qids
SLongQuery* plq = (SLongQuery*)malloc(sizeof(SLongQuery));
plq->qId = qInfo->qId;
plq->startExecTs = qInfo->startExecTs;
taosArrayPush(qids, &plq);
}
// longquery
void* qObtainLongQuery(void* param){
SQueryMgmt* qMgmt = (SQueryMgmt*)param;
if(qMgmt == NULL || qMgmt->qinfoPool == NULL)
return NULL;
SArray* qids = taosArrayInit(4, sizeof(int64_t*));
if(qids == NULL) return NULL;
// Get each item
taosCacheRefresh(qMgmt->qinfoPool, cbFoundItem, qids);
size_t cnt = taosArrayGetSize(qids);
if(cnt == 0) {
taosArrayDestroy(qids);
return NULL;
}
if(cnt > 1)
taosArraySort(qids, compareLongQuery);
return qids;
}
//solve tsdb no block to commit
bool qFixedNoBlock(void* pRepo, void* pMgmt, int32_t longQueryMs) {
SQueryMgmt *pQueryMgmt = pMgmt;
bool fixed = false;
// qid top list
SArray *qids = (SArray*)qObtainLongQuery(pQueryMgmt);
if(qids == NULL) return false;
// kill Query
int64_t now = taosGetTimestampMs();
size_t cnt = taosArrayGetSize(qids);
size_t i;
SLongQuery* plq;
for(i=0; i < cnt; i++) {
plq = (SLongQuery* )taosArrayGetP(qids, i);
if(plq->startExecTs > now) continue;
if(now - plq->startExecTs >= longQueryMs) {
qKillQueryByQId(pMgmt, plq->qId, 500, 10); // wait 50*100 ms
if(tsdbNoProblem(pRepo)) {
fixed = true;
qWarn("QId:0x%"PRIx64" fixed problem after kill this query.", plq->qId);
break;
}
}
}
// free qids
for(i=0; i < cnt; i++) {
free(taosArrayGetP(qids, i));
}
taosArrayDestroy(qids);
return fixed;
}
//solve tsdb no block to commit
bool qSolveCommitNoBlock(void* pRepo, void* pMgmt) {
qWarn("pRepo=%p start solve problem.", pRepo);
if(qFixedNoBlock(pRepo, pMgmt, 10*60*1000)) {
return true;
}
if(qFixedNoBlock(pRepo, pMgmt, 2*60*1000)){
return true;
}
if(qFixedNoBlock(pRepo, pMgmt, 30*1000)){
return true;
}
qWarn("pRepo=%p solve problem failed.", pRepo);
return false;
}

4423
2.0/src/query/src/sql.c
View File

File diff suppressed because it is too large Load Diff

30
2.0/src/query/tests/CMakeLists.txt
View File

@ -1,30 +0,0 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20)
PROJECT(TDengine)
FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest)
FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib /usr/lib64)
FIND_LIBRARY(LIB_GTEST_SHARED_DIR libgtest.so /usr/lib/ /usr/local/lib /usr/lib64)
IF (HEADER_GTEST_INCLUDE_DIR AND (LIB_GTEST_STATIC_DIR OR LIB_GTEST_SHARED_DIR))
MESSAGE(STATUS "gTest library found, build unit test")
# GoogleTest requires at least C++11
SET(CMAKE_CXX_STANDARD 11)
INCLUDE_DIRECTORIES(/usr/include /usr/local/include)
LINK_DIRECTORIES(/usr/lib /usr/local/lib)
INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR})
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
ADD_EXECUTABLE(queryTest ${SOURCE_LIST})
TARGET_LINK_LIBRARIES(queryTest taos query gtest pthread)
ENDIF()
SET_SOURCE_FILES_PROPERTIES(./astTest.cpp PROPERTIES COMPILE_FLAGS -w)
SET_SOURCE_FILES_PROPERTIES(./histogramTest.cpp PROPERTIES COMPILE_FLAGS -w)
SET_SOURCE_FILES_PROPERTIES(./percentileTest.cpp PROPERTIES COMPILE_FLAGS -w)
SET_SOURCE_FILES_PROPERTIES(./resultBufferTest.cpp PROPERTIES COMPILE_FLAGS -w)
SET_SOURCE_FILES_PROPERTIES(./tsBufTest.cpp PROPERTIES COMPILE_FLAGS -w)
SET_SOURCE_FILES_PROPERTIES(./unitTest.cpp PROPERTIES COMPILE_FLAGS -w)
SET_SOURCE_FILES_PROPERTIES(./rangeMergeTest.cpp PROPERTIES COMPILE_FLAGS -w)

637
2.0/src/query/tests/astTest.cpp
View File

@ -1,637 +0,0 @@
#include <gtest/gtest.h>
#include <sys/time.h>
#include <cassert>
#include <iostream>
#include "texpr.h"
#include "tmsg.h"
#include "tsdb.h"
#include "tskiplist.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic ignored "-Wunused-function"
typedef struct ResultObj {
int32_t numOfResult;
char * resultName[64];
} ResultObj;
static void initSchema(SSchema *pSchema, int32_t numOfCols);
static void initSchema_binary(SSchema *schema, int32_t numOfCols);
static SSkipList *createSkipList(SSchema *pSchema, int32_t numOfTags);
static SSkipList *createSkipList_binary(SSchema *pSchema, int32_t numOfTags);
static void dropMeter(SSkipList *pSkipList);
static void Right2LeftTest(SSchema *schema, int32_t numOfCols, SSkipList *pSkipList);
static void Left2RightTest(SSchema *schema, int32_t numOfCols, SSkipList *pSkipList);
static void IllegalExprTest(SSchema *schema, int32_t numOfCols, SSkipList *pSkipList);
static void Left2RightTest_binary(SSchema *schema, int32_t numOfCols, SSkipList *pSkipList);
static void Right2LeftTest_binary(SSchema *schema, int32_t numOfCols, SSkipList *pSkipList);
void setValue(ResultObj *pResult, int32_t num, char **val) {
pResult->numOfResult = num;
for (int32_t i = 0; i < num; ++i) {
pResult->resultName[i] = val[i];
}
}
static void initSchema_binary(SSchema *schema, int32_t numOfCols) {
schema[0].type = TSDB_DATA_TYPE_BINARY;
schema[0].bytes = 8;
strcpy(schema[0].name, "a");
schema[1].type = TSDB_DATA_TYPE_DOUBLE;
schema[1].bytes = 8;
strcpy(schema[1].name, "b");
schema[2].type = TSDB_DATA_TYPE_INT;
schema[2].bytes = 20;
strcpy(schema[2].name, "c");
schema[3].type = TSDB_DATA_TYPE_BIGINT;
schema[3].bytes = 8;
strcpy(schema[3].name, "d");
schema[4].type = TSDB_DATA_TYPE_SMALLINT;
schema[4].bytes = 2;
strcpy(schema[4].name, "e");
schema[5].type = TSDB_DATA_TYPE_TINYINT;
schema[5].bytes = 1;
strcpy(schema[5].name, "f");
schema[6].type = TSDB_DATA_TYPE_FLOAT;
schema[6].bytes = 4;
strcpy(schema[6].name, "g");
schema[7].type = TSDB_DATA_TYPE_BOOL;
schema[7].bytes = 1;
strcpy(schema[7].name, "h");
}
static void initSchema(SSchema *schema, int32_t numOfCols) {
schema[0].type = TSDB_DATA_TYPE_INT;
schema[0].bytes = 8;
strcpy(schema[0].name, "a");
schema[1].type = TSDB_DATA_TYPE_DOUBLE;
schema[1].bytes = 8;
strcpy(schema[1].name, "b");
schema[2].type = TSDB_DATA_TYPE_BINARY;
schema[2].bytes = 20;
strcpy(schema[2].name, "c");
schema[3].type = TSDB_DATA_TYPE_BIGINT;
schema[3].bytes = 8;
strcpy(schema[3].name, "d");
schema[4].type = TSDB_DATA_TYPE_SMALLINT;
schema[4].bytes = 2;
strcpy(schema[4].name, "e");
schema[5].type = TSDB_DATA_TYPE_TINYINT;
schema[5].bytes = 1;
strcpy(schema[5].name, "f");
schema[6].type = TSDB_DATA_TYPE_FLOAT;
schema[6].bytes = 4;
strcpy(schema[6].name, "g");
schema[7].type = TSDB_DATA_TYPE_BOOL;
schema[7].bytes = 1;
strcpy(schema[7].name, "h");
}
// static void addOneNode(SSchema *pSchema, int32_t tagsLen, SSkipList *pSkipList,
// char *meterId, int32_t a, double b, char *c, int64_t d, int16_t e, int8_t f, float g,
// bool h, int32_t numOfTags) {
// STabObj *pMeter = calloc(1, sizeof(STabObj));
// pMeter->numOfTags = numOfTags;
// pMeter->pTagData = calloc(1, tagsLen + TSDB_METER_ID_LEN);
// strcpy(pMeter->meterId, meterId);
//
// char *tags = pMeter->pTagData + TSDB_METER_ID_LEN;
// int32_t offset = 0;
//
// *(int32_t *) tags = a;
//
// offset += pSchema[0].bytes;
// *(double *) (tags + offset) = b;
//
// offset += pSchema[1].bytes;
// memcpy(tags + offset, c, 3);
//
// offset += pSchema[2].bytes;
// *(int64_t *) (tags + offset) = d;
//
// offset += pSchema[3].bytes;
// *(int16_t *) (tags + offset) = e;
//
// offset += pSchema[4].bytes;
// *(int8_t *) (tags + offset) = f;
//
// offset += pSchema[5].bytes;
// *(float *) (tags + offset) = g;
//
// offset += pSchema[6].bytes;
// *(int8_t *) (tags + offset) = h ? 1 : 0;
//
// SSkipListKey pKey = SSkipListCreateKey(pSchema[0].type, tags, pSchema[0].bytes);
// SSkipListPut(pSkipList, pMeter, &pKey, 1);
//}
//
// static void addOneNode_binary(SSchema *pSchema, int32_t tagsLen, SSkipList *pSkipList,
// char *meterId, int32_t a, double b, char *c, int64_t d, int16_t e, int8_t f, float g,
// bool h, int32_t numOfTags) {
// STabObj *pMeter = calloc(1, sizeof(STabObj));
// pMeter->numOfTags = numOfTags;
// pMeter->pTagData = calloc(1, tagsLen + TSDB_METER_ID_LEN);
// strcpy(pMeter->meterId, meterId);
//
// char *tags = pMeter->pTagData + TSDB_METER_ID_LEN;
// int32_t offset = 0;
// memcpy(tags, c, pSchema[0].bytes);
//
// offset += pSchema[0].bytes;
// *(double *) (tags + offset) = b;
//
// offset += pSchema[1].bytes;
// *(int32_t *) (tags + offset) = a;
//
// offset += pSchema[2].bytes;
// *(int64_t *) (tags + offset) = d;
//
// offset += pSchema[3].bytes;
// *(int16_t *) (tags + offset) = e;
//
// offset += pSchema[4].bytes;
// *(int8_t *) (tags + offset) = f;
//
// offset += pSchema[5].bytes;
// *(float *) (tags + offset) = g;
//
// offset += pSchema[6].bytes;
// *(int8_t *) (tags + offset) = h ? 1 : 0;
//
// SSkipListKey pKey = SSkipListCreateKey(pSchema[0].type, tags, pSchema[0].bytes);
// SSkipListPut(pSkipList, pMeter, &pKey, 1);
// SSkipListDestroyKey(&pKey);
//}
// static void dropMeter(SSkipList *pSkipList) {
// SSkipListNode **pRes = NULL;
// int32_t num = SSkipListIterateList(pSkipList, &pRes, NULL, NULL);
// for (int32_t i = 0; i < num; ++i) {
// SSkipListNode *pNode = pRes[i];
// STabObj *pMeter = (STabObj *) pNode->pData;
// free(pMeter->pTagData);
// free(pMeter);
// pNode->pData = NULL;
// }
// free(pRes);
//}
// static SSkipList *createSkipList(SSchema *pSchema, int32_t numOfTags) {
// int32_t tagsLen = 0;
// for (int32_t i = 0; i < numOfTags; ++i) {
// tagsLen += pSchema[i].bytes;
// }
//
// SSkipList *pSkipList = SSkipListCreate(10, pSchema[0].type, 4);
//
// addOneNode(pSchema, tagsLen, pSkipList, "tm0\0", 0, 10.5, "abc", 1000, -10000, -20, 1.0, true, 8);
// addOneNode(pSchema, tagsLen, pSkipList, "tm1\0", 1, 20.5, "def", 1100, -10500, -30, 2.0, false, 8);
// addOneNode(pSchema, tagsLen, pSkipList, "tm2\0", 2, 30.5, "ghi", 1200, -11000, -40, 3.0, true, 8);
// addOneNode(pSchema, tagsLen, pSkipList, "tm3\0", 3, 40.5, "jkl", 1300, -11500, -50, 4.0, false, 8);
// addOneNode(pSchema, tagsLen, pSkipList, "tm4\0", 4, 50.5, "mno", 1400, -12000, -60, 5.0, true, 8);
// addOneNode(pSchema, tagsLen, pSkipList, "tm5\0", 5, 60.5, "pqr", 1500, -12500, -70, 6.0, false, 8);
// addOneNode(pSchema, tagsLen, pSkipList, "tm6\0", 6, 70.5, "stu", 1600, -13000, -80, 7.0, true, 8);
//
// return pSkipList;
//}
//
// static SSkipList *createSkipList_binary(SSchema *pSchema, int32_t numOfTags) {
// int32_t tagsLen = 0;
// for (int32_t i = 0; i < numOfTags; ++i) {
// tagsLen += pSchema[i].bytes;
// }
//
// SSkipList *pSkipList = SSkipListCreate(10, pSchema[0].type, 4);
//
// addOneNode_binary(pSchema, tagsLen, pSkipList, "tm0\0", 0, 10.5, "abc", 1000, -10000, -20, 1.0, true, 8);
// addOneNode_binary(pSchema, tagsLen, pSkipList, "tm1\0", 1, 20.5, "def", 1100, -10500, -30, 2.0, false, 8);
// addOneNode_binary(pSchema, tagsLen, pSkipList, "tm2\0", 2, 30.5, "ghi", 1200, -11000, -40, 3.0, true, 8);
// addOneNode_binary(pSchema, tagsLen, pSkipList, "tm3\0", 3, 40.5, "jkl", 1300, -11500, -50, 4.0, false, 8);
// addOneNode_binary(pSchema, tagsLen, pSkipList, "tm4\0", 4, 50.5, "mno", 1400, -12000, -60, 5.0, true, 8);
// addOneNode_binary(pSchema, tagsLen, pSkipList, "tm5\0", 5, 60.5, "pqr", 1500, -12500, -70, 6.0, false, 8);
// addOneNode_binary(pSchema, tagsLen, pSkipList, "tm6\0", 6, 70.5, "stu", 1600, -13000, -80, 7.0, true, 8);
//
// return pSkipList;
//}
//static void testQueryStr(SSchema *schema, int32_t numOfCols, char *sql, SSkipList *pSkipList, ResultObj *pResult) {
// tExprNode *pExpr = NULL;
// tSQLBinaryExprFromString(&pExpr, schema, numOfCols, sql, strlen(sql));
//
// char str[512] = {0};
// int32_t len = 0;
// if (pExpr == NULL) {
// printf("-----error in parse syntax:%s\n\n", sql);
// assert(pResult == NULL);
// return;
// }
//
// tSQLBinaryExprToString(pExpr, str, &len);
// printf("expr is: %s\n", str);
//
// SArray *result = NULL;
// // tExprTreeTraverse(pExpr, pSkipList, result, SSkipListNodeFilterCallback, &result);
// // printf("the result is:%lld\n", result.num);
// //
// // bool findResult = false;
// // for (int32_t i = 0; i < result.num; ++i) {
// // STabObj *pm = (STabObj *)result.pRes[i];
// // printf("meterid:%s,\t", pm->meterId);
// //
// // for (int32_t j = 0; j < pResult->numOfResult; ++j) {
// // if (strcmp(pm->meterId, pResult->resultName[j]) == 0) {
// // findResult = true;
// // break;
// // }
// // }
// // assert(findResult == true);
// // findResult = false;
// // }
//
// printf("\n\n");
// tExprTreeDestroy(&pExpr, NULL);
//}
#if 0
static void Left2RightTest(SSchema *schema, int32_t numOfCols, SSkipList *pSkipList) {
char str[256] = {0};
char *t0[1] = {"tm0"};
char *t1[1] = {"tm1"};
char *sql = NULL;
ResultObj res = {1, {"tm1"}};
testQueryStr(schema, numOfCols, "a=1", pSkipList, &res);
char *tt[1] = {"tm6"};
setValue(&res, 1, tt);
testQueryStr(schema, numOfCols, "a>=6", pSkipList, &res);
setValue(&res, 1, t0);
testQueryStr(schema, numOfCols, "b<=10.6", pSkipList, &res);
strcpy(str, "c<>'pqr'");
char *t2[6] = {"tm0", "tm1", "tm2", "tm3", "tm4", "tm6"};
setValue(&res, 6, t2);
testQueryStr(schema, numOfCols, str, pSkipList, &res);
strcpy(str, "c='abc'");
setValue(&res, 1, t0);
testQueryStr(schema, numOfCols, str, pSkipList, &res);
char *t3[6] = {"tm1", "tm2", "tm3", "tm4", "tm5", "tm6"};
setValue(&res, 6, t3);
testQueryStr(schema, numOfCols, "d>1050", pSkipList, &res);
char *t4[3] = {"tm4", "tm5", "tm6"};
setValue(&res, 3, t4);
testQueryStr(schema, numOfCols, "g>4.5980765", pSkipList, &res);
char *t5[4] = {"tm0", "tm2", "tm4", "tm6"};
setValue(&res, 4, t5);
testQueryStr(schema, numOfCols, "h=true", pSkipList, &res);
char *t6[3] = {"tm1", "tm3", "tm5"};
setValue(&res, 3, t6);
testQueryStr(schema, numOfCols, "h=0", pSkipList, &res);
sql = "(((b<40)))\0";
char *t7[3] = {"tm0", "tm1", "tm2"};
setValue(&res, 3, t7);
testQueryStr(schema, numOfCols, sql, pSkipList, &res);
sql = "((a=1) or (a=10)) or ((b=12))";
setValue(&res, 1, t1);
testQueryStr(schema, numOfCols, sql, pSkipList, &res);
sql = "((((((a>0 and a<2))) or a=6) or a=3) or (b=50.5)) and h=0";
char *t8[2] = {"tm1", "tm3"};
setValue(&res, 2, t8);
testQueryStr(schema, numOfCols, sql, pSkipList, &res);
char *tf[1] = {"tm6"};
setValue(&res, 1, tf);
testQueryStr(schema, numOfCols, "e = -13000", pSkipList, &res);
char *ft[5] = {"tm0", "tm1", "tm2", "tm3", "tm4"};
setValue(&res, 5, ft);
testQueryStr(schema, numOfCols, "f > -65", pSkipList, &res);
}
void Right2LeftTest(SSchema *schema, int32_t numOfCols, SSkipList *pSkipList) {
ResultObj res = {1, {"tm1"}};
testQueryStr(schema, numOfCols, "((1=a))", pSkipList, &res);
char *t9[2] = {"tm0", "tm1"};
setValue(&res, 2, t9);
testQueryStr(schema, numOfCols, "1>=a", pSkipList, &res);
char *t0[1] = {"tm0"};
setValue(&res, 1, t0);
testQueryStr(schema, numOfCols, "10.6>=b", pSkipList, &res);
char *t10[3] = {"tm1", "tm3", "tm5"};
setValue(&res, 3, t10);
testQueryStr(schema, numOfCols, "0=h", pSkipList, &res);
}
static void IllegalExprTest(SSchema *schema, int32_t numOfCols, SSkipList *pSkipList) {
testQueryStr(schema, numOfCols, "h=", pSkipList, NULL);
testQueryStr(schema, numOfCols, "h<", pSkipList, NULL);
testQueryStr(schema, numOfCols, "a=1 and ", pSkipList, NULL);
testQueryStr(schema, numOfCols, "and or", pSkipList, NULL);
testQueryStr(schema, numOfCols, "and a = 1 or", pSkipList, NULL);
testQueryStr(schema, numOfCols, "(())", pSkipList, NULL);
testQueryStr(schema, numOfCols, "(", pSkipList, NULL);
testQueryStr(schema, numOfCols, "(a", pSkipList, NULL);
testQueryStr(schema, numOfCols, "(a)", pSkipList, NULL);
testQueryStr(schema, numOfCols, "())", pSkipList, NULL);
testQueryStr(schema, numOfCols, "a===1", pSkipList, NULL);
testQueryStr(schema, numOfCols, "a=1 and ", pSkipList, NULL);
}
static void Left2RightTest_binary(SSchema *schema, int32_t numOfCols, SSkipList *pSkipList) {
char str[256] = {0};
char *sql = NULL;
char *t0[1] = {"tm0"};
char *t1[1] = {"tm1"};
ResultObj res = {1, {"tm0"}};
strcpy(str, "a='abc'");
testQueryStr(schema, numOfCols, str, pSkipList, &res);
char *tt[1] = {"tm6"};
setValue(&res, 1, tt);
testQueryStr(schema, numOfCols, "c>=6", pSkipList, &res);
setValue(&res, 1, t0);
testQueryStr(schema, numOfCols, "b<=10.6", pSkipList, &res);
strcpy(str, "a<>'pqr'");
char *t2[6] = {"tm0", "tm1", "tm2", "tm3", "tm4", "tm6"};
setValue(&res, 6, t2);
testQueryStr(schema, numOfCols, str, pSkipList, &res);
strcpy(str, "a='abc'");
setValue(&res, 1, t0);
testQueryStr(schema, numOfCols, str, pSkipList, &res);
char *t3[6] = {"tm1", "tm2", "tm3", "tm4", "tm5", "tm6"};
setValue(&res, 6, t3);
testQueryStr(schema, numOfCols, "d>1050", pSkipList, &res);
char *t4[3] = {"tm4", "tm5", "tm6"};
setValue(&res, 3, t4);
testQueryStr(schema, numOfCols, "g>4.5980765", pSkipList, &res);
char *t5[4] = {"tm0", "tm2", "tm4", "tm6"};
setValue(&res, 4, t5);
testQueryStr(schema, numOfCols, "h=true", pSkipList, &res);
char *t6[3] = {"tm1", "tm3", "tm5"};
setValue(&res, 3, t6);
testQueryStr(schema, numOfCols, "h=0", pSkipList, &res);
sql = "(((b<40)))\0";
char *t7[3] = {"tm0", "tm1", "tm2"};
setValue(&res, 3, t7);
testQueryStr(schema, numOfCols, sql, pSkipList, &res);
sql = "((c=1) or (c=10)) or ((b=12))\0";
setValue(&res, 1, t1);
testQueryStr(schema, numOfCols, sql, pSkipList, &res);
sql = "((((((c>0 and c<2))) or c=6) or c=3) or (b=50.5)) and h=false\0";
char *t8[2] = {"tm1", "tm3"};
setValue(&res, 2, t8);
testQueryStr(schema, numOfCols, sql, pSkipList, &res);
}
static void Right2LeftTest_binary(SSchema *schema, int32_t numOfCols, SSkipList *pSkipList) {
char str[256] = {0};
char *sql = NULL;
char *t0[1] = {"tm0"};
char *t1[1] = {"tm1"};
ResultObj res = {1, {"tm0"}};
strcpy(str, "'abc'=a");
testQueryStr(schema, numOfCols, str, pSkipList, &res);
char *tt[1] = {"tm6"};
setValue(&res, 1, tt);
testQueryStr(schema, numOfCols, "6<=c", pSkipList, &res);
setValue(&res, 1, t0);
testQueryStr(schema, numOfCols, "10.6>=b", pSkipList, &res);
strcpy(str, "'pqr'<>a");
char *t2[6] = {"tm0", "tm1", "tm2", "tm3", "tm4", "tm6"};
setValue(&res, 6, t2);
testQueryStr(schema, numOfCols, str, pSkipList, &res);
}
namespace {
// two level expression tree
tExprNode *createExpr1() {
auto *pLeft = (tExprNode*) calloc(1, sizeof(tExprNode));
pLeft->nodeType = TEXPR_COL_NODE;
pLeft->pSchema = (SSchema*) calloc(1, sizeof(SSchema));
strcpy(pLeft->pSchema->name, "col_a");
pLeft->pSchema->type = TSDB_DATA_TYPE_INT;
pLeft->pSchema->bytes = sizeof(int32_t);
pLeft->pSchema->colId = 1;
auto *pRight = (tExprNode*) calloc(1, sizeof(tExprNode));
pRight->nodeType = TEXPR_VALUE_NODE;
pRight->pVal = (tVariant*) calloc(1, sizeof(tVariant));
pRight->pVal->nType = TSDB_DATA_TYPE_INT;
pRight->pVal->i64 = 12;
auto *pRoot = (tExprNode*) calloc(1, sizeof(tExprNode));
pRoot->nodeType = TEXPR_NODE_EXPR;
pRoot->_node.optr = TSDB_RELATION_EQUAL;
pRoot->_node.pLeft = pLeft;
pRoot->_node.pRight = pRight;
pRoot->_node.hasPK = true;
return pRoot;
}
// thress level expression tree
tExprNode* createExpr2() {
auto *pLeft2 = (tExprNode*) calloc(1, sizeof(tExprNode));
pLeft2->nodeType = TEXPR_COL_NODE;
pLeft2->pSchema = (SSchema*) calloc(1, sizeof(SSchema));
strcpy(pLeft2->pSchema->name, "col_a");
pLeft2->pSchema->type = TSDB_DATA_TYPE_BINARY;
pLeft2->pSchema->bytes = 20;
pLeft2->pSchema->colId = 1;
auto *pRight2 = (tExprNode*) calloc(1, sizeof(tExprNode));
pRight2->nodeType = TEXPR_VALUE_NODE;
pRight2->pVal = (tVariant*) calloc(1, sizeof(tVariant));
pRight2->pVal->nType = TSDB_DATA_TYPE_BINARY;
const char* v = "hello world!";
pRight2->pVal->pz = strdup(v);
pRight2->pVal->nLen = strlen(v);
auto *p1 = (tExprNode*) calloc(1, sizeof(tExprNode));
p1->nodeType = TEXPR_NODE_EXPR;
p1->_node.optr = TSDB_RELATION_LIKE;
p1->_node.pLeft = pLeft2;
p1->_node.pRight = pRight2;
p1->_node.hasPK = false;
auto *pLeft1 = (tExprNode*) calloc(1, sizeof(tExprNode));
pLeft1->nodeType = TEXPR_COL_NODE;
pLeft1->pSchema = (SSchema*) calloc(1, sizeof(SSchema));
strcpy(pLeft1->pSchema->name, "col_b");
pLeft1->pSchema->type = TSDB_DATA_TYPE_DOUBLE;
pLeft1->pSchema->bytes = 8;
pLeft1->pSchema->colId = 99;
auto *pRight1 = (tExprNode*) calloc(1, sizeof(tExprNode));
pRight1->nodeType = TEXPR_VALUE_NODE;
pRight1->pVal = (tVariant*) calloc(1, sizeof(tVariant));
pRight1->pVal->nType = TSDB_DATA_TYPE_DOUBLE;
pRight1->pVal->dKey = 91.99;
auto *p2 = (tExprNode*) calloc(1, sizeof(tExprNode));
p2->nodeType = TEXPR_NODE_EXPR;
p2->_node.optr = TSDB_RELATION_GREATER_EQUAL;
p2->_node.pLeft = pLeft1;
p2->_node.pRight = pRight1;
p2->_node.hasPK = false;
auto *pRoot = (tExprNode*) calloc(1, sizeof(tExprNode));
pRoot->nodeType = TEXPR_NODE_EXPR;
pRoot->_node.optr = TSDB_RELATION_OR;
pRoot->_node.pLeft = p1;
pRoot->_node.pRight = p2;
pRoot->_node.hasPK = true;
return pRoot;
}
void exprSerializeTest1() {
tExprNode* p1 = createExpr1();
SBufferWriter bw = tbufInitWriter(NULL, false);
exprTreeToBinary(&bw, p1);
size_t size = tbufTell(&bw);
ASSERT_TRUE(size > 0);
char* b = tbufGetData(&bw, false);
tExprNode* p2 = exprTreeFromBinary(b, size);
ASSERT_EQ(p1->nodeType, p2->nodeType);
ASSERT_EQ(p2->_node.optr, p1->_node.optr);
ASSERT_EQ(p2->_node.pLeft->nodeType, p1->_node.pLeft->nodeType);
ASSERT_EQ(p2->_node.pRight->nodeType, p1->_node.pRight->nodeType);
SSchema* s1 = p1->_node.pLeft->pSchema;
SSchema* s2 = p2->_node.pLeft->pSchema;
ASSERT_EQ(s2->colId, s1->colId);
ASSERT_EQ(s2->type, s1->type);
ASSERT_EQ(s2->bytes, s1->bytes);
ASSERT_STRCASEEQ(s2->name, s1->name);
tVariant* v1 = p1->_node.pRight->pVal;
tVariant* v2 = p2->_node.pRight->pVal;
ASSERT_EQ(v1->nType, v2->nType);
ASSERT_EQ(v1->i64, v2->i64);
ASSERT_EQ(p1->_node.hasPK, p2->_node.hasPK);
tExprTreeDestroy(&p1, nullptr);
tExprTreeDestroy(&p2, nullptr);
// tbufClose(&bw);
}
void exprSerializeTest2() {
tExprNode* p1 = createExpr2();
SBufferWriter bw = tbufInitWriter(NULL, false);
exprTreeToBinary(&bw, p1);
size_t size = tbufTell(&bw);
ASSERT_TRUE(size > 0);
char* b = tbufGetData(&bw, false);
tExprNode* p2 = exprTreeFromBinary(b, size);
ASSERT_EQ(p1->nodeType, p2->nodeType);
ASSERT_EQ(p2->_node.optr, p1->_node.optr);
ASSERT_EQ(p2->_node.pLeft->nodeType, p1->_node.pLeft->nodeType);
ASSERT_EQ(p2->_node.pRight->nodeType, p1->_node.pRight->nodeType);
tExprNode* c1Left = p1->_node.pLeft;
tExprNode* c2Left = p2->_node.pLeft;
ASSERT_EQ(c1Left->nodeType, c2Left->nodeType);
ASSERT_EQ(c2Left->nodeType, TEXPR_NODE_EXPR);
ASSERT_EQ(c2Left->_node.optr, TSDB_RELATION_LIKE);
ASSERT_STRCASEEQ(c2Left->_node.pLeft->pSchema->name, "col_a");
ASSERT_EQ(c2Left->_node.pRight->nodeType, TEXPR_VALUE_NODE);
ASSERT_STRCASEEQ(c2Left->_node.pRight->pVal->pz, "hello world!");
tExprNode* c1Right = p1->_node.pRight;
tExprNode* c2Right = p2->_node.pRight;
ASSERT_EQ(c1Right->nodeType, c2Right->nodeType);
ASSERT_EQ(c2Right->nodeType, TEXPR_NODE_EXPR);
ASSERT_EQ(c2Right->_node.optr, TSDB_RELATION_GREATER_EQUAL);
ASSERT_EQ(c2Right->_node.pRight->pVal->dKey, 91.99);
ASSERT_EQ(p2->_node.hasPK, true);
tExprTreeDestroy(&p1, nullptr);
tExprTreeDestroy(&p2, nullptr);
// tbufClose(&bw);
}
} // namespace
TEST(testCase, astTest) {
// exprSerializeTest2();
}
#endif
#pragma GCC diagnostic pop

127
2.0/src/query/tests/cSortTest.cpp
View File

@ -1,127 +0,0 @@
#include <gtest/gtest.h>
#include <iostream>
#include "taos.h"
#include "tsdb.h"
#include "qExtbuffer.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wsign-compare"
namespace {
int32_t comp(const void* p1, const void* p2) {
int32_t* x1 = (int32_t*) p1;
int32_t* x2 = (int32_t*) p2;
if (*x1 == *x2) {
return 0;
} else {
return (*x1 > *x2)? 1:-1;
}
}
int32_t comp1(const void* p1, const void* p2) {
int32_t ret = strncmp((char*) p1, (char*) p2, 20);
if (ret == 0) {
return 0;
} else {
return ret > 0 ? 1:-1;
}
}
}
TEST(testCase, colunmnwise_sort_test) {
// void taoscQSort(void** pCols, SSchema* pSchema, int32_t numOfCols, int32_t numOfRows, int32_t index, __compar_fn_t compareFn)
void* pCols[2] = {0};
SSchema s[2] = {{0}};
s[0].type = TSDB_DATA_TYPE_INT;
s[0].bytes = 4;
s[0].colId = 0;
strcpy(s[0].name, "col1");
s[1].type = TSDB_DATA_TYPE_BINARY;
s[1].bytes = 20;
s[1].colId = 1;
strcpy(s[1].name, "col2");
int32_t* p = (int32_t*) calloc(5, sizeof(int32_t));
p[0] = 12;
p[1] = 8;
p[2] = 99;
p[3] = 7;
p[4] = 1;
char* t1 = (char*) calloc(5, 20);
strcpy(t1, "abc");
strcpy(t1 + 20, "def");
strcpy(t1 + 40, "xyz");
strcpy(t1 + 60, "klm");
strcpy(t1 + 80, "hij");
pCols[0] = (char*) p;
pCols[1] = (char*) t1;
taoscQSort(reinterpret_cast<void**>(pCols), s, 2, 5, 0, comp);
int32_t* px = (int32_t*) pCols[0];
ASSERT_EQ(px[0], 1);
ASSERT_EQ(px[1], 7);
ASSERT_EQ(px[2], 8);
ASSERT_EQ(px[3], 12);
ASSERT_EQ(px[4], 99);
char* px1 = (char*) pCols[1];
ASSERT_STRCASEEQ(px1 + 20 * 0, "hij");
ASSERT_STRCASEEQ(px1 + 20 * 1, "klm");
ASSERT_STRCASEEQ(px1 + 20 * 2, "def");
ASSERT_STRCASEEQ(px1 + 20 * 3, "abc");
ASSERT_STRCASEEQ(px1 + 20 * 4, "xyz");
taoscQSort(pCols, s, 2, 5, 1, comp1);
px = (int32_t*) pCols[0];
ASSERT_EQ(px[0], 12);
ASSERT_EQ(px[1], 8);
ASSERT_EQ(px[2], 1);
ASSERT_EQ(px[3], 7);
ASSERT_EQ(px[4], 99);
px1 = (char*) pCols[1];
ASSERT_STRCASEEQ(px1 + 20 * 0, "abc");
ASSERT_STRCASEEQ(px1 + 20 * 1, "def");
ASSERT_STRCASEEQ(px1 + 20 * 2, "hij");
ASSERT_STRCASEEQ(px1 + 20 * 3, "klm");
ASSERT_STRCASEEQ(px1 + 20 * 4, "xyz");
}
TEST(testCase, columnsort_test) {
SSchema field[1] = {
{TSDB_DATA_TYPE_INT, "k", sizeof(int32_t)},
};
const int32_t num = 2000;
int32_t *d = (int32_t *)malloc(sizeof(int32_t) * num);
for (int32_t i = 0; i < num; ++i) {
d[i] = i % 4;
}
const int32_t numOfOrderCols = 1;
int32_t orderColIdx = 0;
SColumnModel *pModel = createColumnModel(field, 1, 1000);
tOrderDescriptor *pDesc = tOrderDesCreate(&orderColIdx, numOfOrderCols, pModel, 1);
tColDataQSort(pDesc, num, 0, num - 1, (char *)d, 1);
for (int32_t i = 0; i < num; ++i) {
printf("%d\t", d[i]);
}
printf("\n");
destroyColumnModel(pModel);
}
#pragma GCC diagnostic pop

145
2.0/src/query/tests/histogramTest.cpp
View File

@ -1,145 +0,0 @@
#include <gtest/gtest.h>
#include <sys/time.h>
#include <cassert>
#include <iostream>
#include "taos.h"
#include "qHistogram.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
namespace {
void doHistogramAddTest() {
SHistogramInfo* pHisto = NULL;
/**
* use arrayList, elapsed time is:
* before:
* 10,000,000 45sec, bin:1000 (-O0) / 17sec. bin:1000, (-O3)
*
* after:
*
*/
struct timeval systemTime;
gettimeofday(&systemTime, NULL);
int64_t st =
(int64_t)systemTime.tv_sec * 1000L + (uint64_t)systemTime.tv_usec / 1000;
for (int32_t i = 0; i < 10000; ++i) {
tHistogramAdd(&pHisto, i);
// tHistogramPrint(pHisto);
}
//
gettimeofday(&systemTime, NULL);
int64_t et =
(int64_t)systemTime.tv_sec * 1000L + (uint64_t)systemTime.tv_usec / 1000;
printf("total elapsed time: %ld\n", et - st);
printf("elements: %d, slot:%d \n", pHisto->numOfElems, pHisto->numOfEntries);
tHistogramPrint(pHisto);
printf("%ld\n", tHistogramSum(pHisto, 1.5));
printf("%ld\n", tHistogramSum(pHisto, 2));
printf("%ld\n", tHistogramSum(pHisto, 3));
printf("%ld\n", tHistogramSum(pHisto, 4));
printf("%ld\n", tHistogramSum(pHisto, 5));
printf("%ld\n", tHistogramSum(pHisto, 6));
for (int32_t i = 399; i < 400; ++i) {
printf("val:%d, %ld\n", i, tHistogramSum(pHisto, i));
}
double ratio[] = {0 / 100, 20.0 / 100, 88.0 / 100, 100 / 100};
double* res = tHistogramUniform(pHisto, ratio, 4);
for (int32_t i = 0; i < 4; ++i) {
printf("%f\n", res[i]);
}
SHistogramInfo* pHisto1 = NULL;
for (int32_t i = (90000 - 1); i >= 80000; --i) {
tHistogramAdd(&pHisto1, i);
}
tHistogramPrint(pHisto1);
SHistogramInfo* pRes = tHistogramMerge(pHisto1, pHisto, MAX_HISTOGRAM_BIN);
assert(pRes->numOfElems == pHisto->numOfElems + pHisto1->numOfElems);
tHistogramPrint(pRes);
tHistogramDestroy(&pHisto);
tHistogramDestroy(&pHisto1);
tHistogramDestroy(&pRes);
free(res);
}
void doHistogramRepeatTest() {
SHistogramInfo* pHisto = NULL;
struct timeval systemTime;
gettimeofday(&systemTime, NULL);
int64_t st =
(int64_t)systemTime.tv_sec * 1000L + (uint64_t)systemTime.tv_usec / 1000;
for (int32_t i = 0; i < 1000; ++i) {
tHistogramAdd(&pHisto, -24 + i);
// tHistogramPrint(pHisto);
}
tHistogramDestroy(&pHisto);
}
}
/* test validate the names for table/database */
TEST(testCase, histogram_binary_search) {
SHistogramInfo* pHisto = tHistogramCreate(MAX_HISTOGRAM_BIN);
pHisto->numOfEntries = 10;
for (int32_t i = 0; i < 10; ++i) {
pHisto->elems[i].num = 1;
pHisto->elems[i].val = i;
}
int32_t idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 1);
assert(idx == 1);
idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 9);
assert(idx == 9);
idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 20);
assert(idx == 10);
idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, -1);
assert(idx == 0);
idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 3.9);
assert(idx == 4);
free(pHisto);
}
TEST(testCase, histogram_add) {
doHistogramAddTest();
doHistogramRepeatTest();
}
TEST(testCase, heapsort) {
// int32_t num = 20;
//
// SHeapEntry* pEntry = tHeapCreate(num);
//
// for(int32_t i=0; i<num; ++i) {
// pEntry[i].val = num - 1 - i;
// }
//
// tHeapSort(pEntry, num);
//
// for(int32_t i=0; i<num; ++i) {
// printf("%lf, ", pEntry[i].val);
// }
//
// printf("\n");
//
// free(pEntry);
}
#pragma GCC diagnostic pop

89
2.0/src/query/tests/patternMatchTest.cpp
View File

@ -1,89 +0,0 @@
#include <gtest/gtest.h>
#include <sys/time.h>
#include <cassert>
#include <iostream>
#include "qAggMain.h"
#include "tcompare.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
TEST(testCase, patternMatchTest) {
SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER;
const char* str = "abcdef";
int32_t ret = patternMatch("a%b%", str, strlen(str), &info);
EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
str = "tm01";
ret = patternMatch("tm__", str, strlen(str), &info);
EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
str = "tkm1";
ret = patternMatch("t%m1", str, strlen(str), &info);
EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
str = "tkm1";
ret = patternMatch("%m1", str, strlen(str), &info);
EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
str = "";
ret = patternMatch("%_", str, strlen(str), &info);
EXPECT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH);
str = "1";
ret = patternMatch("%__", str, strlen(str), &info);
EXPECT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH);
str = "";
ret = patternMatch("%", str, strlen(str), &info);
EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
str = " ";
ret = patternMatch("_", str, strlen(str), &info);
EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
str = "!";
ret = patternMatch("%_", str, strlen(str), &info);
EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
str = "abcdefg";
ret = patternMatch("abc%fg", str, strlen(str), &info);
EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
str = "abcdefgabcdeju";
ret = patternMatch("abc%fg", str, 7, &info);
EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
str = "abcdefgabcdeju";
ret = patternMatch("abc%f_", str, 6, &info);
EXPECT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH);
str = "abcdefgabcdeju";
ret = patternMatch("abc%f_", str, 1, &info); // pattern string is longe than the size
EXPECT_EQ(ret, TSDB_PATTERN_NOMATCH);
str = "abcdefgabcdeju";
ret = patternMatch("ab", str, 2, &info);
EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
str = "abcdefgabcdeju";
ret = patternMatch("a%", str, 2, &info);
EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
str = "abcdefgabcdeju";
ret = patternMatch("a__", str, 2, &info);
EXPECT_EQ(ret, TSDB_PATTERN_NOMATCH);
str = "carzero";
ret = patternMatch("%o", str, strlen(str), &info);
EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
str = "19";
ret = patternMatch("%9", str, 2, &info);
EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
}
#pragma GCC diagnostic pop

260
2.0/src/query/tests/percentileTest.cpp
View File

@ -1,260 +0,0 @@
#include <gtest/gtest.h>
#include <iostream>
#include "qResultbuf.h"
#include "taos.h"
#include "taosdef.h"
#include "qPercentile.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
namespace {
tMemBucket *createBigIntDataBucket(int32_t start, int32_t end) {
tMemBucket *pBucket = tMemBucketCreate(sizeof(int64_t), TSDB_DATA_TYPE_BIGINT, start, end);
for (int32_t i = start; i <= end; ++i) {
int64_t val = i;
tMemBucketPut(pBucket, &val, 1);
}
return pBucket;
}
tMemBucket *createIntDataBucket(int32_t start, int32_t end) {
tMemBucket *pBucket = tMemBucketCreate(sizeof(int32_t), TSDB_DATA_TYPE_INT, start, end);
for (int32_t i = start; i <= end; ++i) {
int32_t val = i;
tMemBucketPut(pBucket, &val, 1);
}
return pBucket;
}
tMemBucket *createDoubleDataBucket(int32_t start, int32_t end) {
tMemBucket *pBucket = tMemBucketCreate(sizeof(double), TSDB_DATA_TYPE_DOUBLE, start, end);
for (int32_t i = start; i <= end; ++i) {
double val = i;
int32_t ret = tMemBucketPut(pBucket, &val, 1);
if (ret != 0) {
printf("value out of range:%f", val);
}
}
return pBucket;
}
tMemBucket *createUnsignedDataBucket(int32_t start, int32_t end, int32_t type) {
tMemBucket *pBucket = tMemBucketCreate(tDataTypes[type].bytes, type, start, end);
for (int32_t i = start; i <= end; ++i) {
uint64_t k = i;
int32_t ret = tMemBucketPut(pBucket, &k, 1);
if (ret != 0) {
printf("value out of range:%" PRId64, k);
}
}
return pBucket;
}
void intDataTest() {
printf("running %s\n", __FUNCTION__);
tMemBucket *pBucket = NULL;
double result = 0.;
pBucket = createIntDataBucket(0, 0);
result = getPercentile(pBucket, 0);
ASSERT_DOUBLE_EQ(result, 0);
tMemBucketDestroy(pBucket);
pBucket = createIntDataBucket(0, 1);
result = getPercentile(pBucket, 100);
ASSERT_DOUBLE_EQ(result, 1);
result = getPercentile(pBucket, 0);
ASSERT_DOUBLE_EQ(result, 0);
tMemBucketDestroy(pBucket);
pBucket = createIntDataBucket(-1, 1);
result = getPercentile(pBucket, 50);
ASSERT_DOUBLE_EQ(result, 0);
result = getPercentile(pBucket, 0);
ASSERT_DOUBLE_EQ(result, -1);
result = getPercentile(pBucket, 75);
ASSERT_DOUBLE_EQ(result, 0.5);
result = getPercentile(pBucket, 100);
ASSERT_DOUBLE_EQ(result, 1);
tMemBucketDestroy(pBucket);
pBucket = createIntDataBucket(0, 99999);
result = getPercentile(pBucket, 50);
ASSERT_DOUBLE_EQ(result, 49999.5);
tMemBucketDestroy(pBucket);
}
void bigintDataTest() {
printf("running %s\n", __FUNCTION__);
tMemBucket *pBucket = NULL;
double result = 0.0;
pBucket = createBigIntDataBucket(-1000, 1000);
result = getPercentile(pBucket, 50);
ASSERT_DOUBLE_EQ(result, 0.);
tMemBucketDestroy(pBucket);
pBucket = createBigIntDataBucket(-10000, 10000);
result = getPercentile(pBucket, 100);
ASSERT_DOUBLE_EQ(result, 10000.0);
tMemBucketDestroy(pBucket);
pBucket = createBigIntDataBucket(-10000, 10000);
result = getPercentile(pBucket, 75);
ASSERT_DOUBLE_EQ(result, 5000.0);
tMemBucketDestroy(pBucket);
}
void doubleDataTest() {
printf("running %s\n", __FUNCTION__);
tMemBucket *pBucket = NULL;
double result = 0;
pBucket = createDoubleDataBucket(-10, 10);
result = getPercentile(pBucket, 0);
ASSERT_DOUBLE_EQ(result, -10.0);
printf("result is: %lf\n", result);
tMemBucketDestroy(pBucket);
pBucket = createDoubleDataBucket(-100000, 100000);
result = getPercentile(pBucket, 25);
ASSERT_DOUBLE_EQ(result, -50000);
printf("result is: %lf\n", result);
tMemBucketDestroy(pBucket);
pBucket = createDoubleDataBucket(-100000, 100000);
result = getPercentile(pBucket, 50);
ASSERT_DOUBLE_EQ(result, 0);
tMemBucketDestroy(pBucket);
pBucket = createDoubleDataBucket(-100000, 100000);
result = getPercentile(pBucket, 75);
ASSERT_DOUBLE_EQ(result, 50000);
tMemBucketDestroy(pBucket);
pBucket = createDoubleDataBucket(-100000, 100000);
result = getPercentile(pBucket, 100);
ASSERT_DOUBLE_EQ(result, 100000.0);
printf("result is: %lf\n", result);
tMemBucketDestroy(pBucket);
}
/*
* large data test, we employ 0.1billion double data to calculated the percentile
* which is 800MB data
*/
void largeDataTest() {
printf("running : %s\n", __FUNCTION__);
tMemBucket *pBucket = NULL;
double result = 0;
struct timeval tv;
gettimeofday(&tv, NULL);
int64_t start = tv.tv_sec;
printf("start time: %" PRId64 "\n", tv.tv_sec);
pBucket = createDoubleDataBucket(0, 100000000);
result = getPercentile(pBucket, 50);
ASSERT_DOUBLE_EQ(result, 50000000);
gettimeofday(&tv, NULL);
printf("total elapsed time: %" PRId64 " sec.", -start + tv.tv_sec);
printf("the result of %d is: %lf\n", 50, result);
tMemBucketDestroy(pBucket);
}
void qsortTest() {
printf("running : %s\n", __FUNCTION__);
SSchema field[1] = {
{TSDB_DATA_TYPE_INT, "k", sizeof(int32_t)},
};
const int32_t num = 2000;
int32_t *d = (int32_t *)malloc(sizeof(int32_t) * num);
for (int32_t i = 0; i < num; ++i) {
d[i] = i % 4;
}
const int32_t numOfOrderCols = 1;
int32_t orderColIdx = 0;
SColumnModel * pModel = createColumnModel(field, 1, 1000);
tOrderDescriptor *pDesc = tOrderDesCreate(&orderColIdx, numOfOrderCols, pModel, 1);
tColDataQSort(pDesc, num, 0, num - 1, (char *)d, 1);
for (int32_t i = 0; i < num; ++i) {
printf("%d\t", d[i]);
}
printf("\n");
destroyColumnModel(pModel);
}
void unsignedDataTest() {
printf("running %s\n", __FUNCTION__);
tMemBucket *pBucket = NULL;
double result = 0.0;
pBucket = createUnsignedDataBucket(0, 1000, TSDB_DATA_TYPE_UINT);
result = getPercentile(pBucket, 50);
ASSERT_DOUBLE_EQ(result, 500.0);
tMemBucketDestroy(pBucket);
pBucket = createUnsignedDataBucket(0, 10000, TSDB_DATA_TYPE_UBIGINT);
result = getPercentile(pBucket, 100);
ASSERT_DOUBLE_EQ(result, 10000.0);
result = getPercentile(pBucket, 0);
ASSERT_DOUBLE_EQ(result, 0.0);
result = getPercentile(pBucket, 50);
ASSERT_DOUBLE_EQ(result, 5000);
result = getPercentile(pBucket, 75);
ASSERT_DOUBLE_EQ(result, 7500);
tMemBucketDestroy(pBucket);
}
} // namespace
TEST(testCase, percentileTest) {
// qsortTest();
intDataTest();
bigintDataTest();
doubleDataTest();
unsignedDataTest();
largeDataTest();
}
#pragma GCC diagnostic pop

370
2.0/src/query/tests/rangeMergeTest.cpp
View File

@ -1,370 +0,0 @@
#include <gtest/gtest.h>
#include <iostream>
#include "qResultbuf.h"
#include "taos.h"
#include "taosdef.h"
#include "qFilter.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
extern "C" {
extern void* filterInitRangeCtx(int32_t type, int32_t options);
extern int32_t filterGetRangeNum(void* h, int32_t* num);
extern int32_t filterGetRangeRes(void* h, SFilterRange *ra);
extern int32_t filterFreeRangeCtx(void* h);
extern int32_t filterAddRange(void* h, SFilterRange* ra, int32_t optr);
}
namespace {
void intDataTest() {
printf("running %s\n", __FUNCTION__);
int32_t asize = 0;
SFilterRange ra[10] = {0};
int64_t *s =NULL;
int64_t *e =NULL;
int64_t s0[3] = {-100, 1, 3};
int64_t e0[3] = {0 , 2, 4};
int64_t s1[3] = {INT64_MIN, 0 , 3};
int64_t e1[3] = {100 , 50, 4};
int64_t s2[5] = {1 , 3 , 10,30,70};
int64_t e2[5] = {10, 100, 20,50,120};
int64_t s3[3] = {1 , 20 , 5};
int64_t e3[3] = {10, 100, 25};
int64_t s4[2] = {10, 0};
int64_t e4[2] = {20, 5};
int64_t s5[3] = {0, 6 ,7};
int64_t e5[3] = {4, 10,20};
int64_t rs[10];
int64_t re[10];
int32_t num = 0;
void *h = NULL;
s = s0;
e = e0;
asize = sizeof(s0)/sizeof(s[0]);
memset(ra, 0, sizeof(ra));
h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0);
for (int32_t i = 0; i < asize; ++i) {
ra[0].s = s[i];
ra[0].e = e[i];
filterAddRange(h, ra, TSDB_RELATION_AND);
}
filterGetRangeNum(h, &num);
ASSERT_EQ(num, 0);
filterFreeRangeCtx(h);
memset(ra, 0, sizeof(ra));
h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0);
for (int32_t i = 0; i < asize; ++i) {
ra[0].s = s[i];
ra[0].e = e[i];
filterAddRange(h, ra, TSDB_RELATION_OR);
}
filterGetRangeNum(h, &num);
ASSERT_EQ(num, 3);
filterGetRangeRes(h, ra);
ASSERT_EQ(ra[0].s, -100);
ASSERT_EQ(ra[0].e, 0);
ASSERT_EQ(ra[1].s, 1);
ASSERT_EQ(ra[1].e, 2);
ASSERT_EQ(ra[2].s, 3);
ASSERT_EQ(ra[2].e, 4);
filterFreeRangeCtx(h);
memset(ra, 0, sizeof(ra));
h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, FI_OPTION_TIMESTAMP);
for (int32_t i = 0; i < asize; ++i) {
ra[0].s = s[i];
ra[0].e = e[i];
filterAddRange(h, ra, TSDB_RELATION_OR);
}
filterGetRangeNum(h, &num);
ASSERT_EQ(num, 1);
filterGetRangeRes(h, ra);
ASSERT_EQ(ra[0].s, -100);
ASSERT_EQ(ra[0].e, 4);
filterFreeRangeCtx(h);
s = s1;
e = e1;
asize = sizeof(s1)/sizeof(s[0]);
memset(ra, 0, sizeof(ra));
h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0);
for (int32_t i = 0; i < asize; ++i) {
ra[0].s = s[i];
ra[0].e = e[i];
filterAddRange(h, ra, TSDB_RELATION_AND);
}
filterGetRangeNum(h, &num);
ASSERT_EQ(num, 1);
filterGetRangeRes(h, ra);
ASSERT_EQ(ra[0].s, 3);
ASSERT_EQ(ra[0].e, 4);
filterFreeRangeCtx(h);
memset(ra, 0, sizeof(ra));
h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0);
for (int32_t i = 0; i < asize; ++i) {
ra[0].s = s[i];
ra[0].e = e[i];
filterAddRange(h, ra, TSDB_RELATION_OR);
}
filterGetRangeNum(h, &num);
ASSERT_EQ(num, 1);
filterGetRangeRes(h, ra);
ASSERT_EQ(ra[0].s, INT64_MIN);
ASSERT_EQ(ra[0].e, 100);
filterFreeRangeCtx(h);
s = s2;
e = e2;
asize = sizeof(s2)/sizeof(s[0]);
memset(ra, 0, sizeof(ra));
h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0);
for (int32_t i = 0; i < asize; ++i) {
ra[0].s = s[i];
ra[0].e = e[i];
filterAddRange(h, ra, TSDB_RELATION_AND);
}
filterGetRangeNum(h, &num);
ASSERT_EQ(num, 0);
filterFreeRangeCtx(h);
memset(ra, 0, sizeof(ra));
h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0);
for (int32_t i = 0; i < asize; ++i) {
ra[0].s = s[i];
ra[0].e = e[i];
filterAddRange(h, ra, TSDB_RELATION_OR);
}
filterGetRangeNum(h, &num);
ASSERT_EQ(num, 1);
filterGetRangeRes(h, ra);
ASSERT_EQ(ra[0].s, 1);
ASSERT_EQ(ra[0].e, 120);
filterFreeRangeCtx(h);
memset(ra, 0, sizeof(ra));
h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0);
for (int32_t i = 0; i < asize; ++i) {
ra[0].s = s[i];
ra[0].e = e[i];
filterAddRange(h, ra, i % 2 ? TSDB_RELATION_OR : TSDB_RELATION_AND);
}
filterGetRangeNum(h, &num);
ASSERT_EQ(num, 0);
filterFreeRangeCtx(h);
memset(ra, 0, sizeof(ra));
h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0);
for (int32_t i = 0; i < asize; ++i) {
ra[0].s = s[i];
ra[0].e = e[i];
filterAddRange(h, ra, i % 2 ? TSDB_RELATION_AND : TSDB_RELATION_OR);
}
filterGetRangeNum(h, &num);
ASSERT_EQ(num, 1);
filterGetRangeRes(h, ra);
ASSERT_EQ(ra[0].s, 70);
ASSERT_EQ(ra[0].e, 120);
filterFreeRangeCtx(h);
s = s3;
e = e3;
asize = sizeof(s3)/sizeof(s[0]);
memset(ra, 0, sizeof(ra));
h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0);
for (int32_t i = 0; i < asize; ++i) {
ra[0].s = s[i];
ra[0].e = e[i];
filterAddRange(h, ra, TSDB_RELATION_AND);
}
filterGetRangeNum(h, &num);
ASSERT_EQ(num, 0);
filterFreeRangeCtx(h);
memset(ra, 0, sizeof(ra));
h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0);
for (int32_t i = 0; i < asize; ++i) {
ra[0].s = s[i];
ra[0].e = e[i];
filterAddRange(h, ra, TSDB_RELATION_OR);
}
filterGetRangeNum(h, &num);
ASSERT_EQ(num, 1);
filterGetRangeRes(h, ra);
ASSERT_EQ(ra[0].s, 1);
ASSERT_EQ(ra[0].e, 100);
filterFreeRangeCtx(h);
s = s4;
e = e4;
asize = sizeof(s4)/sizeof(s[0]);
memset(ra, 0, sizeof(ra));
h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0);
for (int32_t i = 0; i < asize; ++i) {
ra[0].s = s[i];
ra[0].e = e[i];
filterAddRange(h, ra, TSDB_RELATION_AND);
}
filterGetRangeNum(h, &num);
ASSERT_EQ(num, 0);
filterFreeRangeCtx(h);
memset(ra, 0, sizeof(ra));
h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0);
for (int32_t i = 0; i < asize; ++i) {
ra[0].s = s[i];
ra[0].e = e[i];
filterAddRange(h, ra, TSDB_RELATION_OR);
}
filterGetRangeNum(h, &num);
ASSERT_EQ(num, 2);
filterGetRangeRes(h, ra);
ASSERT_EQ(ra[0].s, 0);
ASSERT_EQ(ra[0].e, 5);
ASSERT_EQ(ra[1].s, 10);
ASSERT_EQ(ra[1].e, 20);
filterFreeRangeCtx(h);
s = s5;
e = e5;
asize = sizeof(s5)/sizeof(s[0]);
memset(ra, 0, sizeof(ra));
h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0);
for (int32_t i = 0; i < asize; ++i) {
ra[0].s = s[i];
ra[0].e = e[i];
filterAddRange(h, ra, TSDB_RELATION_AND);
}
filterGetRangeNum(h, &num);
ASSERT_EQ(num, 0);
filterFreeRangeCtx(h);
memset(ra, 0, sizeof(ra));
h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0);
for (int32_t i = 0; i < asize; ++i) {
ra[0].s = s[i];
ra[0].e = e[i];
filterAddRange(h, ra, TSDB_RELATION_OR);
}
filterGetRangeNum(h, &num);
ASSERT_EQ(num, 2);
filterGetRangeRes(h, ra);
ASSERT_EQ(ra[0].s, 0);
ASSERT_EQ(ra[0].e, 4);
ASSERT_EQ(ra[1].s, 6);
ASSERT_EQ(ra[1].e, 20);
filterFreeRangeCtx(h);
memset(ra, 0, sizeof(ra));
h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0);
for (int32_t i = 0; i < asize; ++i) {
ra[0].s = s[i];
ra[0].e = e[i];
filterAddRange(h, ra, (i == (asize -1)) ? TSDB_RELATION_AND : TSDB_RELATION_OR);
}
filterGetRangeNum(h, &num);
ASSERT_EQ(num, 1);
filterGetRangeRes(h, ra);
ASSERT_EQ(ra[0].s, 7);
ASSERT_EQ(ra[0].e, 10);
filterFreeRangeCtx(h);
int64_t s6[2] = {0, 4};
int64_t e6[2] = {4, 6};
s = s6;
e = e6;
asize = sizeof(s6)/sizeof(s[0]);
memset(ra, 0, sizeof(ra));
h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0);
for (int32_t i = 0; i < asize; ++i) {
ra[0].eflag = 1;
ra[1].sflag = 4;
ra[i].s = s[i];
ra[i].e = e[i];
filterAddRange(h, ra + i, TSDB_RELATION_AND);
}
filterGetRangeNum(h, &num);
ASSERT_EQ(num, 1);
filterFreeRangeCtx(h);
memset(ra, 0, sizeof(ra));
h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0);
for (int32_t i = 0; i < asize; ++i) {
ra[0].eflag = 1;
ra[1].sflag = 1;
ra[i].s = s[i];
ra[i].e = e[i];
filterAddRange(h, ra + i, TSDB_RELATION_OR);
}
filterGetRangeNum(h, &num);
ASSERT_EQ(num, 2);
ASSERT_EQ(ra[0].s, 0);
ASSERT_EQ(ra[0].e, 4);
ASSERT_EQ(ra[0].eflag, 1);
ASSERT_EQ(ra[1].s, 4);
ASSERT_EQ(ra[1].e, 6);
ASSERT_EQ(ra[1].sflag, 1);
filterFreeRangeCtx(h);
}
} // namespace
TEST(testCase, rangeMergeTest) {
intDataTest();
}
#pragma GCC diagnostic pop

166
2.0/src/query/tests/resultBufferTest.cpp
View File

@ -1,166 +0,0 @@
#include <gtest/gtest.h>
#include <cassert>
#include <iostream>
#include "qResultbuf.h"
#include "taos.h"
#include "tsdb.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
namespace {
// simple test
void simpleTest() {
SDiskbasedBuf* pResultBuf = NULL;
int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1024, 4096, 1);
int32_t pageId = 0;
int32_t groupId = 0;
tFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId);
ASSERT_TRUE(pBufPage != NULL);
ASSERT_EQ(getTotalBufSize(pResultBuf), 1024);
SIDList list = getDataBufPagesIdList(pResultBuf, groupId);
ASSERT_EQ(taosArrayGetSize(list), 1);
ASSERT_EQ(getNumOfResultBufGroupId(pResultBuf), 1);
releaseBufPage(pResultBuf, pBufPage);
tFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId);
tFilePage* t = getBufPage(pResultBuf, pageId);
ASSERT_TRUE(t == pBufPage1);
tFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId);
tFilePage* t1 = getBufPage(pResultBuf, pageId);
ASSERT_TRUE(t1 == pBufPage2);
tFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId);
tFilePage* t2 = getBufPage(pResultBuf, pageId);
ASSERT_TRUE(t2 == pBufPage3);
tFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId);
tFilePage* t3 = getBufPage(pResultBuf, pageId);
ASSERT_TRUE(t3 == pBufPage4);
tFilePage* pBufPage5 = getNewDataBuf(pResultBuf, groupId, &pageId);
tFilePage* t4 = getBufPage(pResultBuf, pageId);
ASSERT_TRUE(t4 == pBufPage5);
destroyResultBuf(pResultBuf);
}
void writeDownTest() {
SDiskbasedBuf* pResultBuf = NULL;
int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1024, 4*1024, 1);
int32_t pageId = 0;
int32_t writePageId = 0;
int32_t groupId = 0;
int32_t nx = 12345;
tFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId);
ASSERT_TRUE(pBufPage != NULL);
*(int32_t*)(pBufPage->data) = nx;
writePageId = pageId;
releaseBufPage(pResultBuf, pBufPage);
tFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId);
tFilePage* t1 = getBufPage(pResultBuf, pageId);
ASSERT_TRUE(t1 == pBufPage1);
ASSERT_TRUE(pageId == 1);
tFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId);
tFilePage* t2 = getBufPage(pResultBuf, pageId);
ASSERT_TRUE(t2 == pBufPage2);
ASSERT_TRUE(pageId == 2);
tFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId);
tFilePage* t3 = getBufPage(pResultBuf, pageId);
ASSERT_TRUE(t3 == pBufPage3);
ASSERT_TRUE(pageId == 3);
tFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId);
tFilePage* t4 = getBufPage(pResultBuf, pageId);
ASSERT_TRUE(t4 == pBufPage4);
ASSERT_TRUE(pageId == 4);
releaseBufPage(pResultBuf, t4);
// flush the written page to disk, and read it out again
tFilePage* pBufPagex = getBufPage(pResultBuf, writePageId);
ASSERT_EQ(*(int32_t*)pBufPagex->data, nx);
SArray* pa = getDataBufPagesIdList(pResultBuf, groupId);
ASSERT_EQ(taosArrayGetSize(pa), 5);
destroyResultBuf(pResultBuf);
}
void recyclePageTest() {
SDiskbasedBuf* pResultBuf = NULL;
int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1024, 4*1024, 1);
int32_t pageId = 0;
int32_t writePageId = 0;
int32_t groupId = 0;
int32_t nx = 12345;
tFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId);
ASSERT_TRUE(pBufPage != NULL);
releaseBufPage(pResultBuf, pBufPage);
tFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId);
tFilePage* t1 = getBufPage(pResultBuf, pageId);
ASSERT_TRUE(t1 == pBufPage1);
ASSERT_TRUE(pageId == 1);
tFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId);
tFilePage* t2 = getBufPage(pResultBuf, pageId);
ASSERT_TRUE(t2 == pBufPage2);
ASSERT_TRUE(pageId == 2);
tFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId);
tFilePage* t3 = getBufPage(pResultBuf, pageId);
ASSERT_TRUE(t3 == pBufPage3);
ASSERT_TRUE(pageId == 3);
tFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId);
tFilePage* t4 = getBufPage(pResultBuf, pageId);
ASSERT_TRUE(t4 == pBufPage4);
ASSERT_TRUE(pageId == 4);
releaseBufPage(pResultBuf, t4);
tFilePage* pBufPage5 = getNewDataBuf(pResultBuf, groupId, &pageId);
tFilePage* t5 = getBufPage(pResultBuf, pageId);
ASSERT_TRUE(t5 == pBufPage5);
ASSERT_TRUE(pageId == 5);
// flush the written page to disk, and read it out again
tFilePage* pBufPagex = getBufPage(pResultBuf, writePageId);
*(int32_t*)(pBufPagex->data) = nx;
writePageId = pageId; // update the data
releaseBufPage(pResultBuf, pBufPagex);
tFilePage* pBufPagex1 = getBufPage(pResultBuf, 1);
SArray* pa = getDataBufPagesIdList(pResultBuf, groupId);
ASSERT_EQ(taosArrayGetSize(pa), 6);
destroyResultBuf(pResultBuf);
}
} // namespace
TEST(testCase, resultBufferTest) {
srand(time(NULL));
simpleTest();
writeDownTest();
recyclePageTest();
}
#pragma GCC diagnostic pop

518
2.0/src/query/tests/tsBufTest.cpp
View File

@ -1,518 +0,0 @@
#include "os.h"
#include <gtest/gtest.h>
#include <cassert>
#include <iostream>
#include "qTsbuf.h"
#include "taos.h"
#include "tsdb.h"
#include "ttoken.h"
#include "tutil.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wunused-but-set-variable"
namespace {
/**
*
* @param num total number
* @param step gap between two consecutive ts
* @return
*/
int64_t* createTsList(int32_t num, int64_t start, int32_t step) {
int64_t* pList = (int64_t*)malloc(num * sizeof(int64_t));
for (int64_t i = 0; i < num; ++i) {
pList[i] = start + i * step;
}
return pList;
}
// simple test
void simpleTest() {
STSBuf* pTSBuf = tsBufCreate(true, TSDB_ORDER_ASC);
// write 10 ts points
int32_t num = 10;
tVariant t = {0};
t.nType = TSDB_DATA_TYPE_BIGINT;
t.i64 = 1;
int64_t* list = createTsList(10, 10000000, 30);
tsBufAppend(pTSBuf, 0, &t, (const char*)list, num * sizeof(int64_t));
EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC);
EXPECT_EQ(pTSBuf->tsData.len, sizeof(int64_t) * num);
EXPECT_EQ(tVariantCompare(&pTSBuf->block.tag, &t), 0);
EXPECT_EQ(pTSBuf->numOfGroups, 1);
tsBufFlush(pTSBuf);
EXPECT_EQ(pTSBuf->tsData.len, 0);
EXPECT_EQ(pTSBuf->block.numOfElem, num);
tsBufDestroy(pTSBuf);
free(list);
}
// one large list of ts, the ts list need to be split into several small blocks
void largeTSTest() {
STSBuf* pTSBuf = tsBufCreate(true, TSDB_ORDER_ASC);
// write 10 ts points
int32_t num = 1000000;
tVariant t = {0};
t.nType = TSDB_DATA_TYPE_BIGINT;
t.i64 = 1;
int64_t* list = createTsList(num, 10000000, 30);
tsBufAppend(pTSBuf, 0, &t, (const char*)list, num * sizeof(int64_t));
// the data has been flush to disk, no data in cache
EXPECT_EQ(pTSBuf->tsData.len, 0);
EXPECT_EQ(tVariantCompare(&pTSBuf->block.tag, &t), 0);
EXPECT_EQ(pTSBuf->numOfGroups, 1);
EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC);
tsBufFlush(pTSBuf);
EXPECT_EQ(pTSBuf->tsData.len, 0);
EXPECT_EQ(pTSBuf->block.numOfElem, num);
tsBufDestroy(pTSBuf);
free(list);
}
void multiTagsTest() {
STSBuf* pTSBuf = tsBufCreate(true, TSDB_ORDER_ASC);
int32_t num = 10000;
tVariant t = {0};
t.nType = TSDB_DATA_TYPE_BIGINT;
int64_t start = 10000000;
int32_t numOfTags = 50;
int32_t step = 30;
for (int32_t i = 0; i < numOfTags; ++i) {
int64_t* list = createTsList(num, start, step);
t.i64 = i;
tsBufAppend(pTSBuf, 0, &t, (const char*)list, num * sizeof(int64_t));
free(list);
start += step * num;
}
EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC);
EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t));
EXPECT_EQ(pTSBuf->block.tag.i64, numOfTags - 1);
EXPECT_EQ(pTSBuf->numOfGroups, 1);
tsBufFlush(pTSBuf);
EXPECT_EQ(pTSBuf->tsData.len, 0);
EXPECT_EQ(pTSBuf->block.numOfElem, num);
tsBufDestroy(pTSBuf);
}
void multiVnodeTagsTest() {
STSBuf* pTSBuf = tsBufCreate(true, TSDB_ORDER_ASC);
int32_t num = 10000;
int64_t start = 10000000;
int32_t numOfTags = 50;
int32_t step = 30;
// 2000 vnodes
for (int32_t j = 0; j < 20; ++j) {
// vnodeId:0
start = 10000000;
tVariant t = {0};
t.nType = TSDB_DATA_TYPE_BIGINT;
for (int32_t i = 0; i < numOfTags; ++i) {
int64_t* list = createTsList(num, start, step);
t.i64 = i;
tsBufAppend(pTSBuf, j, &t, (const char*)list, num * sizeof(int64_t));
free(list);
start += step * num;
}
EXPECT_EQ(pTSBuf->numOfGroups, j + 1);
}
EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC);
EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t));
EXPECT_EQ(pTSBuf->block.tag.i64, numOfTags - 1);
EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t));
EXPECT_EQ(pTSBuf->block.tag.i64, numOfTags - 1);
tsBufFlush(pTSBuf);
EXPECT_EQ(pTSBuf->tsData.len, 0);
EXPECT_EQ(pTSBuf->block.numOfElem, num);
tsBufDestroy(pTSBuf);
}
void loadDataTest() {
STSBuf* pTSBuf = tsBufCreate(true, TSDB_ORDER_ASC);
int32_t num = 10000;
int64_t oldStart = 10000000;
int32_t numOfTags = 50;
int32_t step = 30;
int32_t numOfVnode = 200;
// 10000 vnodes
for (int32_t j = 0; j < numOfVnode; ++j) {
// vnodeId:0
int64_t start = 10000000;
tVariant t = {0};
t.nType = TSDB_DATA_TYPE_BIGINT;
for (int32_t i = 0; i < numOfTags; ++i) {
int64_t* list = createTsList(num, start, step);
t.i64 = i;
tsBufAppend(pTSBuf, j, &t, (const char*)list, num * sizeof(int64_t));
printf("%d - %" PRIu64 "\n", i, list[0]);
free(list);
start += step * num;
}
EXPECT_EQ(pTSBuf->numOfGroups, j + 1);
}
EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC);
EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t));
EXPECT_EQ(pTSBuf->block.tag.i64, numOfTags - 1);
EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t));
EXPECT_EQ(pTSBuf->block.tag.i64, numOfTags - 1);
tsBufFlush(pTSBuf);
EXPECT_EQ(pTSBuf->tsData.len, 0);
EXPECT_EQ(pTSBuf->block.numOfElem, num);
// create from exists file
STSBuf* pNewBuf = tsBufCreateFromFile(pTSBuf->path, false);
EXPECT_EQ(pNewBuf->tsOrder, pTSBuf->tsOrder);
EXPECT_EQ(pNewBuf->numOfGroups, numOfVnode);
EXPECT_EQ(pNewBuf->fileSize, pTSBuf->fileSize);
EXPECT_EQ(pNewBuf->pData[0].info.offset, pTSBuf->pData[0].info.offset);
EXPECT_EQ(pNewBuf->pData[0].info.numOfBlocks, pTSBuf->pData[0].info.numOfBlocks);
EXPECT_EQ(pNewBuf->pData[0].info.compLen, pTSBuf->pData[0].info.compLen);
EXPECT_STREQ(pNewBuf->path, pTSBuf->path);
tsBufResetPos(pNewBuf);
int64_t s = taosGetTimestampUs();
printf("start:%" PRIu64 "\n", s);
int32_t x = 0;
while (tsBufNextPos(pNewBuf)) {
STSElem elem = tsBufGetElem(pNewBuf);
if (++x == 100000000) {
break;
}
// printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag, elem.ts);
}
int64_t e = taosGetTimestampUs();
printf("end:%" PRIu64 ", elapsed:%" PRIu64 ", total obj:%d\n", e, e - s, x);
tsBufDestroy(pTSBuf);
tsBufDestroy(pNewBuf);
}
void randomIncTsTest() {}
void TSTraverse() {
// 10000 vnodes
int32_t num = 200000;
int64_t oldStart = 10000000;
int32_t numOfTags = 3;
int32_t step = 30;
int32_t numOfVnode = 2;
STSBuf* pTSBuf = tsBufCreate(true, TSDB_ORDER_ASC);
for (int32_t j = 0; j < numOfVnode; ++j) {
// vnodeId:0
int64_t start = 10000000;
tVariant t = {0};
t.nType = TSDB_DATA_TYPE_BIGINT;
for (int32_t i = 0; i < numOfTags; ++i) {
int64_t* list = createTsList(num, start, step);
t.i64 = i;
tsBufAppend(pTSBuf, j, &t, (const char*)list, num * sizeof(int64_t));
printf("%d - %d - %" PRIu64 ", %" PRIu64 "\n", j, i, list[0], list[num - 1]);
free(list);
start += step * num;
list = createTsList(num, start, step);
tsBufAppend(pTSBuf, j, &t, (const char*)list, num * sizeof(int64_t));
printf("%d - %d - %" PRIu64 ", %" PRIu64 "\n", j, i, list[0], list[num - 1]);
free(list);
start += step * num;
}
EXPECT_EQ(pTSBuf->numOfGroups, j + 1);
}
tsBufResetPos(pTSBuf);
////////////////////////////////////////////////////////////////////////////////////////
// reverse traverse
int64_t s = taosGetTimestampUs();
printf("start:%" PRIu64 "\n", s);
pTSBuf->cur.order = TSDB_ORDER_DESC;
// complete reverse traverse
int32_t x = 0;
while (tsBufNextPos(pTSBuf)) {
STSElem elem = tsBufGetElem(pTSBuf);
// printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag, elem.ts);
}
// specify the data block with vnode and tags value
tsBufResetPos(pTSBuf);
pTSBuf->cur.order = TSDB_ORDER_DESC;
int32_t startVnode = 1;
int32_t startTag = 2;
tVariant t = {0};
t.nType = TSDB_DATA_TYPE_BIGINT;
t.i64 = startTag;
tsBufGetElemStartPos(pTSBuf, startVnode, &t);
int32_t totalOutput = 10;
while (1) {
STSElem elem = tsBufGetElem(pTSBuf);
printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.id, elem.tag->i64, elem.ts);
if (!tsBufNextPos(pTSBuf)) {
break;
}
if (--totalOutput <= 0) {
totalOutput = 10;
startTag -= 1;
t.i64 = startTag;
tsBufGetElemStartPos(pTSBuf, startVnode, &t);
if (startTag == 0) {
startVnode -= 1;
startTag = 3;
}
if (startVnode < 0) {
break;
}
}
}
/////////////////////////////////////////////////////////////////////////////////
// traverse
pTSBuf->cur.order = TSDB_ORDER_ASC;
tsBufResetPos(pTSBuf);
// complete forwards traverse
while (tsBufNextPos(pTSBuf)) {
STSElem elem = tsBufGetElem(pTSBuf);
// printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag, elem.ts);
}
// specify the data block with vnode and tags value
tsBufResetPos(pTSBuf);
pTSBuf->cur.order = TSDB_ORDER_ASC;
startVnode = 1;
startTag = 2;
t.i64 = startTag;
tsBufGetElemStartPos(pTSBuf, startVnode, &t);
totalOutput = 10;
while (1) {
STSElem elem = tsBufGetElem(pTSBuf);
printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.id, elem.tag->i64, elem.ts);
if (!tsBufNextPos(pTSBuf)) {
break;
}
if (--totalOutput <= 0) {
totalOutput = 10;
startTag -= 1;
t.i64 = startTag;
tsBufGetElemStartPos(pTSBuf, startVnode, &t);
if (startTag < 0) {
startVnode -= 1;
startTag = 3;
}
if (startVnode < 0) {
break;
}
}
}
tsBufDestroy(pTSBuf);
}
void performanceTest() {}
void emptyTagTest() {}
void invalidFileTest() {
const char* cmd = "touch /tmp/test";
// create empty file
system(cmd);
STSBuf* pNewBuf = tsBufCreateFromFile("/tmp/test", true);
EXPECT_TRUE(pNewBuf == NULL);
tsBufDestroy(pNewBuf);
pNewBuf = tsBufCreateFromFile("/tmp/911", true);
EXPECT_TRUE(pNewBuf == NULL);
tsBufDestroy(pNewBuf);
}
void mergeDiffVnodeBufferTest() {
STSBuf* pTSBuf1 = tsBufCreate(true, TSDB_ORDER_ASC);
STSBuf* pTSBuf2 = tsBufCreate(true, TSDB_ORDER_ASC);
int32_t step = 30;
int32_t num = 1000;
int32_t numOfTags = 10;
tVariant t = {0};
t.nType = TSDB_DATA_TYPE_BIGINT;
// vnodeId:0
int64_t start = 10000000;
for (int32_t i = 0; i < numOfTags; ++i) {
int64_t* list = createTsList(num, start, step);
t.i64 = i;
tsBufAppend(pTSBuf1, 1, &t, (const char*)list, num * sizeof(int64_t));
tsBufAppend(pTSBuf2, 9, &t, (const char*)list, num * sizeof(int64_t));
free(list);
start += step * num;
}
tsBufFlush(pTSBuf2);
tsBufMerge(pTSBuf1, pTSBuf2);
EXPECT_EQ(pTSBuf1->numOfGroups, 2);
EXPECT_EQ(pTSBuf1->numOfTotal, numOfTags * 2 * num);
tsBufDisplay(pTSBuf1);
tsBufDestroy(pTSBuf2);
tsBufDestroy(pTSBuf1);
}
void mergeIdenticalVnodeBufferTest() {
STSBuf* pTSBuf1 = tsBufCreate(true, TSDB_ORDER_ASC);
STSBuf* pTSBuf2 = tsBufCreate(true, TSDB_ORDER_ASC);
tVariant t = {0};
t.nType = TSDB_DATA_TYPE_BIGINT;
int32_t step = 30;
int32_t num = 1000;
int32_t numOfTags = 10;
// vnodeId:0
int64_t start = 10000000;
for (int32_t i = 0; i < numOfTags; ++i) {
int64_t* list = createTsList(num, start, step);
t.i64 = i;
tsBufAppend(pTSBuf1, 12, &t, (const char*)list, num * sizeof(int64_t));
free(list);
start += step * num;
}
for (int32_t i = numOfTags; i < numOfTags * 2; ++i) {
int64_t* list = createTsList(num, start, step);
t.i64 = i;
tsBufAppend(pTSBuf2, 77, &t, (const char*)list, num * sizeof(int64_t));
free(list);
start += step * num;
}
tsBufFlush(pTSBuf2);
tsBufMerge(pTSBuf1, pTSBuf2);
EXPECT_EQ(pTSBuf1->numOfGroups, 2);
EXPECT_EQ(pTSBuf1->numOfTotal, numOfTags * 2 * num);
tsBufResetPos(pTSBuf1);
int32_t count = 0;
while (tsBufNextPos(pTSBuf1)) {
STSElem elem = tsBufGetElem(pTSBuf1);
if (count++ < numOfTags * num) {
EXPECT_EQ(elem.id, 12);
} else {
EXPECT_EQ(elem.id, 77);
}
printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.id, elem.tag->i64, elem.ts);
}
tsBufDestroy(pTSBuf1);
tsBufDestroy(pTSBuf2);
}
} // namespace
//TODO add binary tag value test case
TEST(testCase, tsBufTest) {
simpleTest();
largeTSTest();
multiTagsTest();
multiVnodeTagsTest();
loadDataTest();
invalidFileTest();
// randomIncTsTest();
TSTraverse();
mergeDiffVnodeBufferTest();
mergeIdenticalVnodeBufferTest();
}
#pragma GCC diagnostic pop

914
2.0/src/query/tests/unitTest.cpp
View File

@ -1,914 +0,0 @@
#include <gtest/gtest.h>
#include <iostream>
#include "taos.h"
#include "tsdb.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wunused-but-set-variable"
#pragma GCC diagnostic ignored "-Wsign-compare"
#include "../../client/inc/tscUtil.h"
#include "tutil.h"
#include "tvariant.h"
#include "ttokendef.h"
namespace {
int32_t testValidateName(char* name) {
SStrToken token = {0};
token.z = name;
token.n = strlen(name);
token.type = 0;
tGetToken(name, &token.type);
return tscValidateName(&token);
}
}
static void _init_tvariant_bool(tVariant* t) {
t->i64 = TSDB_FALSE;
t->nType = TSDB_DATA_TYPE_BOOL;
}
static void _init_tvariant_tinyint(tVariant* t) {
t->i64 = -27;
t->nType = TSDB_DATA_TYPE_TINYINT;
}
static void _init_tvariant_int(tVariant* t) {
t->i64 = -23997659;
t->nType = TSDB_DATA_TYPE_INT;
}
static void _init_tvariant_bigint(tVariant* t) {
t->i64 = -3333333333333;
t->nType = TSDB_DATA_TYPE_BIGINT;
}
static void _init_tvariant_float(tVariant* t) {
t->dKey = -8991212199.8987878776;
t->nType = TSDB_DATA_TYPE_FLOAT;
}
static void _init_tvariant_binary(tVariant* t) {
tVariantDestroy(t);
t->pz = (char*)calloc(1, 20); //"2e3");
t->nType = TSDB_DATA_TYPE_BINARY;
strcpy(t->pz, "2e5");
t->nLen = strlen(t->pz);
}
static void _init_tvariant_nchar(tVariant* t) {
tVariantDestroy(t);
t->wpz = (wchar_t*)calloc(1, 20 * TSDB_NCHAR_SIZE);
t->nType = TSDB_DATA_TYPE_NCHAR;
wcscpy(t->wpz, L"-2000000.8765");
t->nLen = twcslen(t->wpz);
}
int main(int argc, char** argv) {
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
/* test validate the names for table/database */
TEST(testCase, db_table_name) {
char t01[] = "abc";
EXPECT_EQ(testValidateName(t01), TSDB_CODE_SUCCESS);
char t02[] = "'abc'";
EXPECT_EQ(testValidateName(t02), TSDB_CODE_SUCCESS);
char t1[] = "abc.def";
EXPECT_EQ(testValidateName(t1), TSDB_CODE_SUCCESS);
printf("%s\n", t1);
char t2[] = "'abc.def'";
EXPECT_EQ(testValidateName(t2), TSDB_CODE_SUCCESS);
printf("%s\n", t2);
char t3[] = "'abc'.def";
EXPECT_EQ(testValidateName(t3), TSDB_CODE_SUCCESS);
printf("%s\n", t3);
char t4[] = "'abc'.'def'";
EXPECT_EQ(testValidateName(t4), TSDB_CODE_SUCCESS);
char t5[] = "table.'def'";
EXPECT_EQ(testValidateName(t5), TSDB_CODE_TSC_INVALID_OPERATION);
char t6[] = "'table'.'def'";
EXPECT_EQ(testValidateName(t6), TSDB_CODE_TSC_INVALID_OPERATION);
char t7[] = "'_ab1234'.'def'";
EXPECT_EQ(testValidateName(t7), TSDB_CODE_SUCCESS);
printf("%s\n", t7);
char t8[] = "'_ab&^%1234'.'def'";
EXPECT_EQ(testValidateName(t8), TSDB_CODE_TSC_INVALID_OPERATION);
char t9[] = "'_123'.'gtest中文'";
EXPECT_EQ(testValidateName(t9), TSDB_CODE_TSC_INVALID_OPERATION);
char t10[] = "abc.'gtest中文'";
EXPECT_EQ(testValidateName(t10), TSDB_CODE_TSC_INVALID_OPERATION);
char t10_1[] = "abc.'中文gtest'";
EXPECT_EQ(testValidateName(t10_1), TSDB_CODE_TSC_INVALID_OPERATION);
char t11[] = "'192.168.0.1'.abc";
EXPECT_EQ(testValidateName(t11), TSDB_CODE_TSC_INVALID_OPERATION);
char t12[] = "192.168.0.1.abc";
EXPECT_EQ(testValidateName(t12), TSDB_CODE_TSC_INVALID_OPERATION);
char t13[] = "abc.";
EXPECT_EQ(testValidateName(t13), TSDB_CODE_TSC_INVALID_OPERATION);
char t14[] = ".abc";
EXPECT_EQ(testValidateName(t14), TSDB_CODE_TSC_INVALID_OPERATION);
char t15[] = ".'abc'";
EXPECT_EQ(testValidateName(t15), TSDB_CODE_TSC_INVALID_OPERATION);
char t16[] = ".abc'";
EXPECT_EQ(testValidateName(t16), TSDB_CODE_TSC_INVALID_OPERATION);
char t17[] = "123a.\"abc\"";
EXPECT_EQ(testValidateName(t17), TSDB_CODE_TSC_INVALID_OPERATION);
printf("%s\n", t17);
char t18[] = "a.\"abc\"";
EXPECT_EQ(testValidateName(t18), TSDB_CODE_SUCCESS);
printf("%s\n", t18);
char t19[] = "'_ab1234'.'def'.'ab123'";
EXPECT_EQ(testValidateName(t19), TSDB_CODE_TSC_INVALID_OPERATION);
char t20[] = "'_ab1234*&^'";
EXPECT_EQ(testValidateName(t20), TSDB_CODE_TSC_INVALID_OPERATION);
char t21[] = "'1234_abc'";
EXPECT_EQ(testValidateName(t21), TSDB_CODE_TSC_INVALID_OPERATION);
// =======Containing capital letters=================
char t30[] = "ABC";
EXPECT_EQ(testValidateName(t30), TSDB_CODE_SUCCESS);
char t31[] = "'ABC'";
EXPECT_EQ(testValidateName(t31), TSDB_CODE_SUCCESS);
char t32[] = "ABC.def";
EXPECT_EQ(testValidateName(t32), TSDB_CODE_SUCCESS);
char t33[] = "'ABC.def";
EXPECT_EQ(testValidateName(t33), TSDB_CODE_TSC_INVALID_OPERATION);
char t33_0[] = "abc.DEF'";
EXPECT_EQ(testValidateName(t33_0), TSDB_CODE_TSC_INVALID_OPERATION);
char t34[] = "'ABC.def'";
//int32_t tmp0 = testValidateName(t34);
EXPECT_EQ(testValidateName(t34), TSDB_CODE_SUCCESS);
char t35[] = "'ABC'.def";
EXPECT_EQ(testValidateName(t35), TSDB_CODE_SUCCESS);
char t36[] = "'ABC'.'DEF'";
EXPECT_EQ(testValidateName(t36), TSDB_CODE_SUCCESS);
char t37[] = "abc.'DEF'";
EXPECT_EQ(testValidateName(t37), TSDB_CODE_SUCCESS);
char t37_1[] = "abc.'_123DEF'";
EXPECT_EQ(testValidateName(t37_1), TSDB_CODE_SUCCESS);
char t38[] = "'abc'.'DEF'";
EXPECT_EQ(testValidateName(t38), TSDB_CODE_SUCCESS);
// do not use key words
char t39[] = "table.'DEF'";
EXPECT_EQ(testValidateName(t39), TSDB_CODE_TSC_INVALID_OPERATION);
char t40[] = "'table'.'DEF'";
EXPECT_EQ(testValidateName(t40), TSDB_CODE_TSC_INVALID_OPERATION);
char t41[] = "'_abXYZ1234'.'deFF'";
EXPECT_EQ(testValidateName(t41), TSDB_CODE_SUCCESS);
char t42[] = "'_abDEF&^%1234'.'DIef'";
EXPECT_EQ(testValidateName(t42), TSDB_CODE_TSC_INVALID_OPERATION);
char t43[] = "'_123'.'Gtest中文'";
EXPECT_EQ(testValidateName(t43), TSDB_CODE_TSC_INVALID_OPERATION);
char t44[] = "'aABC'.'Gtest中文'";
EXPECT_EQ(testValidateName(t44), TSDB_CODE_TSC_INVALID_OPERATION);
char t45[] = "'ABC'.";
EXPECT_EQ(testValidateName(t45), TSDB_CODE_TSC_INVALID_OPERATION);
char t46[] = ".'ABC'";
EXPECT_EQ(testValidateName(t46), TSDB_CODE_TSC_INVALID_OPERATION);
char t47[] = "a.\"aTWc\"";
EXPECT_EQ(testValidateName(t47), TSDB_CODE_SUCCESS);
// ================has space =================
char t60[] = " ABC ";
EXPECT_EQ(testValidateName(t60), TSDB_CODE_TSC_INVALID_OPERATION);
char t60_1[] = " ABC ";
EXPECT_EQ(testValidateName(t60_1), TSDB_CODE_TSC_INVALID_OPERATION);
char t61[] = "' ABC '";
EXPECT_EQ(testValidateName(t61), TSDB_CODE_TSC_INVALID_OPERATION);
char t61_1[] = "' ABC '";
EXPECT_EQ(testValidateName(t61_1), TSDB_CODE_TSC_INVALID_OPERATION);
char t62[] = " ABC . def ";
EXPECT_EQ(testValidateName(t62), TSDB_CODE_TSC_INVALID_OPERATION);
char t63[] = "' ABC . def ";
EXPECT_EQ(testValidateName(t63), TSDB_CODE_TSC_INVALID_OPERATION);
char t63_0[] = " abc . DEF ' ";
EXPECT_EQ(testValidateName(t63_0), TSDB_CODE_TSC_INVALID_OPERATION);
char t64[] = " ' ABC . def ' ";
//int32_t tmp1 = testValidateName(t64);
EXPECT_EQ(testValidateName(t64), TSDB_CODE_TSC_INVALID_OPERATION);
char t65[] = " ' ABC '. def ";
EXPECT_EQ(testValidateName(t65), TSDB_CODE_TSC_INVALID_OPERATION);
char t66[] = "' ABC '.' DEF '";
EXPECT_EQ(testValidateName(t66), TSDB_CODE_TSC_INVALID_OPERATION);
char t67[] = "abc . ' DEF '";
EXPECT_EQ(testValidateName(t67), TSDB_CODE_TSC_INVALID_OPERATION);
char t68[] = "' abc '.' DEF '";
EXPECT_EQ(testValidateName(t68), TSDB_CODE_TSC_INVALID_OPERATION);
// do not use key words
char t69[] = "table.'DEF'";
EXPECT_EQ(testValidateName(t69), TSDB_CODE_TSC_INVALID_OPERATION);
char t70[] = "'table'.'DEF'";
EXPECT_EQ(testValidateName(t70), TSDB_CODE_TSC_INVALID_OPERATION);
char t71[] = "'_abXYZ1234 '.' deFF '";
EXPECT_EQ(testValidateName(t71), TSDB_CODE_TSC_INVALID_OPERATION);
char t72[] = "'_abDEF&^%1234'.' DIef'";
EXPECT_EQ(testValidateName(t72), TSDB_CODE_TSC_INVALID_OPERATION);
char t73[] = "'_123'.' Gtest中文'";
EXPECT_EQ(testValidateName(t73), TSDB_CODE_TSC_INVALID_OPERATION);
char t74[] = "' aABC'.'Gtest中文'";
EXPECT_EQ(testValidateName(t74), TSDB_CODE_TSC_INVALID_OPERATION);
char t75[] = "' ABC '.";
EXPECT_EQ(testValidateName(t75), TSDB_CODE_TSC_INVALID_OPERATION);
char t76[] = ".' ABC'";
EXPECT_EQ(testValidateName(t76), TSDB_CODE_TSC_INVALID_OPERATION);
char t77[] = " a . \"aTWc\" ";
EXPECT_EQ(testValidateName(t77), TSDB_CODE_TSC_INVALID_OPERATION);
char t78[] = " a.\"aTWc \"";
EXPECT_EQ(testValidateName(t78), TSDB_CODE_TSC_INVALID_OPERATION);
// ===============muti string by space ===================
// There's no such case.
//char t160[] = "A BC";
//EXPECT_EQ(testValidateName(t160), TSDB_CODE_TSC_INVALID_OPERATION);
//printf("end:%s\n", t160);
// There's no such case.
//char t161[] = "' A BC '";
//EXPECT_EQ(testValidateName(t161), TSDB_CODE_TSC_INVALID_OPERATION);
char t162[] = " AB C . de f ";
EXPECT_EQ(testValidateName(t162), TSDB_CODE_TSC_INVALID_OPERATION);
char t163[] = "' AB C . de f ";
EXPECT_EQ(testValidateName(t163), TSDB_CODE_TSC_INVALID_OPERATION);
char t163_0[] = " ab c . DE F ' ";
EXPECT_EQ(testValidateName(t163_0), TSDB_CODE_TSC_INVALID_OPERATION);
char t164[] = " ' AB C . de f ' ";
//int32_t tmp2 = testValidateName(t164);
EXPECT_EQ(testValidateName(t164), TSDB_CODE_TSC_INVALID_OPERATION);
char t165[] = " ' A BC '. de f ";
EXPECT_EQ(testValidateName(t165), TSDB_CODE_TSC_INVALID_OPERATION);
char t166[] = "' AB C '.' DE F '";
EXPECT_EQ(testValidateName(t166), TSDB_CODE_TSC_INVALID_OPERATION);
char t167[] = "ab c . ' D EF '";
EXPECT_EQ(testValidateName(t167), TSDB_CODE_TSC_INVALID_OPERATION);
char t168[] = "' a bc '.' DE F '";
EXPECT_EQ(testValidateName(t168), TSDB_CODE_TSC_INVALID_OPERATION);
}
/* test parse time function */
TEST(testCase, parse_time) {
taos_options(TSDB_OPTION_TIMEZONE, "GMT-8");
char t1[] = "2018-1-1 1:1:1.952798";
char t13[] = "1970-1-1 0:0:0";
int64_t time = 0, time1 = 0;
taosParseTime(t1, &time, strlen(t1), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, 1514739661952);
taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, timezone * MILLISECOND_PER_SECOND);
char t2[] = "2018-1-1T1:1:1.952Z";
taosParseTime(t2, &time, strlen(t2), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, 1514739661952 + 28800000);
char t3[] = "2018-1-1 1:01:01.952";
taosParseTime(t3, &time, strlen(t3), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, 1514739661952);
char t4[] = "2018-1-1 1:01:01.9";
char t5[] = "2018-1-1 1:01:1.900";
char t6[] = "2018-01-01 1:1:1.90";
char t7[] = "2018-01-01 01:01:01.9";
char t8[] = "2018-01-01 01:01:01.9007865";
taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI, 0);
taosParseTime(t5, &time1, strlen(t5), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, time1);
taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI, 0);
taosParseTime(t6, &time1, strlen(t6), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, time1);
taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI, 0);
taosParseTime(t7, &time1, strlen(t7), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, time1);
taosParseTime(t5, &time, strlen(t5), TSDB_TIME_PRECISION_MILLI, 0);
taosParseTime(t8, &time1, strlen(t8), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, time1);
char t9[] = "2017-4-3 1:1:2.980";
char t10[] = "2017-4-3T2:1:2.98+9:00";
taosParseTime(t9, &time, strlen(t9), TSDB_TIME_PRECISION_MILLI, 0);
taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, time1);
char t11[] = "2017-4-3T2:1:2.98+09:00";
taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI, 0);
taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, time1);
char t12[] = "2017-4-3T2:1:2.98+0900";
taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI, 0);
taosParseTime(t12, &time1, strlen(t12), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, time1);
taos_options(TSDB_OPTION_TIMEZONE, "UTC");
taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, 0);
taos_options(TSDB_OPTION_TIMEZONE, "Asia/Shanghai");
char t14[] = "1970-1-1T0:0:0Z";
taosParseTime(t14, &time, strlen(t14), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, 0);
char t40[] = "1970-1-1 0:0:0.999999999";
taosParseTime(t40, &time, strlen(t40), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, 999 + timezone * MILLISECOND_PER_SECOND);
char t41[] = "1997-1-1 0:0:0.999999999";
taosParseTime(t41, &time, strlen(t41), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, 852048000999);
char t42[] = "1997-1-1T0:0:0.999999999Z";
taosParseTime(t42, &time, strlen(t42), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, 852048000999 - timezone * MILLISECOND_PER_SECOND);
// "%Y-%m-%d %H:%M:%S" format with TimeZone appendix is also treated as legal
// and TimeZone will be processed
char t60[] = "2017-4-3 1:1:2.980";
char t61[] = "2017-4-3 2:1:2.98+9:00";
taosParseTime(t60, &time, strlen(t60), TSDB_TIME_PRECISION_MILLI, 0);
taosParseTime(t61, &time1, strlen(t61), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, time1);
char t62[] = "2017-4-3 2:1:2.98+09:00";
taosParseTime(t62, &time, strlen(t62), TSDB_TIME_PRECISION_MILLI, 0);
taosParseTime(t61, &time1, strlen(t61), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, time1);
char t63[] = "2017-4-3 2:1:2.98+0900";
taosParseTime(t63, &time, strlen(t63), TSDB_TIME_PRECISION_MILLI, 0);
taosParseTime(t62, &time1, strlen(t62), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, time1);
char t64[] = "2017-4-2 17:1:2.98Z";
taosParseTime(t63, &time, strlen(t63), TSDB_TIME_PRECISION_MILLI, 0);
taosParseTime(t64, &time1, strlen(t64), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, time1);
// "%Y-%m-%d%H:%M:%S" format with TimeZone appendix is also treated as legal
// and TimeZone will be processed
char t80[] = "2017-4-51:1:2.980";
char t81[] = "2017-4-52:1:2.98+9:00";
taosParseTime(t80, &time, strlen(t80), TSDB_TIME_PRECISION_MILLI, 0);
taosParseTime(t81, &time1, strlen(t81), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, time1);
char t82[] = "2017-4-52:1:2.98+09:00";
taosParseTime(t82, &time, strlen(t82), TSDB_TIME_PRECISION_MILLI, 0);
taosParseTime(t81, &time1, strlen(t81), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, time1);
char t83[] = "2017-4-52:1:2.98+0900";
taosParseTime(t83, &time, strlen(t83), TSDB_TIME_PRECISION_MILLI, 0);
taosParseTime(t82, &time1, strlen(t82), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, time1);
char t84[] = "2017-4-417:1:2.98Z";
taosParseTime(t83, &time, strlen(t83), TSDB_TIME_PRECISION_MILLI, 0);
taosParseTime(t84, &time1, strlen(t84), TSDB_TIME_PRECISION_MILLI, 0);
EXPECT_EQ(time, time1);
////////////////////////////////////////////////////////////////////
// illegal timestamp format
char t15[] = "2017-12-33 0:0:0";
EXPECT_EQ(taosParseTime(t15, &time, strlen(t15), TSDB_TIME_PRECISION_MILLI, 0), -1);
char t16[] = "2017-12-31 99:0:0";
EXPECT_EQ(taosParseTime(t16, &time, strlen(t16), TSDB_TIME_PRECISION_MILLI, 0), -1);
char t17[] = "2017-12-31T9:0:0";
EXPECT_EQ(taosParseTime(t17, &time, strlen(t17), TSDB_TIME_PRECISION_MILLI, 0), -1);
char t18[] = "2017-12-31T9:0:0.Z";
EXPECT_EQ(taosParseTime(t18, &time, strlen(t18), TSDB_TIME_PRECISION_MILLI, 0), -1);
char t19[] = "2017-12-31 9:0:0.-1";
EXPECT_EQ(taosParseTime(t19, &time, strlen(t19), TSDB_TIME_PRECISION_MILLI, 0), -1);
char t20[] = "2017-12-31 9:0:0.1+12:99";
EXPECT_EQ(taosParseTime(t20, &time, strlen(t20), TSDB_TIME_PRECISION_MILLI, 0), -1);
char t21[] = "2017-12-31T9:0:0.1+12:99";
EXPECT_EQ(taosParseTime(t21, &time, strlen(t21), TSDB_TIME_PRECISION_MILLI, 0), -1);
char t22[] = "2017-12-31 9:0:0.1+13:1";
EXPECT_EQ(taosParseTime(t22, &time, strlen(t22), TSDB_TIME_PRECISION_MILLI, 0), 0);
char t23[] = "2017-12-31T9:0:0.1+13:1";
EXPECT_EQ(taosParseTime(t23, &time, strlen(t23), TSDB_TIME_PRECISION_MILLI, 0), 0);
char t24[] = "2017-12-31T9:0:0.1+13:001";
EXPECT_EQ(taosParseTime(t24, &time, strlen(t24), TSDB_TIME_PRECISION_MILLI, 0), -1);
char t25[] = "2017-12-31T9:0:0.1+13:00abc";
EXPECT_EQ(taosParseTime(t25, &time, strlen(t25), TSDB_TIME_PRECISION_MILLI, 0), -1);
char t26[] = "2017-12-31T9:0:0.1+13001";
EXPECT_EQ(taosParseTime(t26, &time, strlen(t26), TSDB_TIME_PRECISION_MILLI, 0), -1);
char t27[] = "2017-12-31T9:0:0.1+1300abc";
EXPECT_EQ(taosParseTime(t27, &time, strlen(t27), TSDB_TIME_PRECISION_MILLI, 0), -1);
char t28[] = "2017-12-31T9:0:0Z+12:00";
EXPECT_EQ(taosParseTime(t28, &time, strlen(t28), TSDB_TIME_PRECISION_MILLI, 0), -1);
char t29[] = "2017-12-31T9:0:0.123Z+12:00";
EXPECT_EQ(taosParseTime(t29, &time, strlen(t29), TSDB_TIME_PRECISION_MILLI, 0), -1);
char t65[] = "2017-12-31 9:0:0.1+13:001";
EXPECT_EQ(taosParseTime(t65, &time, strlen(t65), TSDB_TIME_PRECISION_MILLI, 0), -1);
char t66[] = "2017-12-31 9:0:0.1+13:00abc";
EXPECT_EQ(taosParseTime(t66, &time, strlen(t66), TSDB_TIME_PRECISION_MILLI, 0), -1);
char t67[] = "2017-12-31 9:0:0.1+13001";
EXPECT_EQ(taosParseTime(t67, &time, strlen(t67), TSDB_TIME_PRECISION_MILLI, 0), -1);
char t68[] = "2017-12-31 9:0:0.1+1300abc";
EXPECT_EQ(taosParseTime(t68, &time, strlen(t68), TSDB_TIME_PRECISION_MILLI, 0), -1);
char t69[] = "2017-12-31 9:0:0Z+12:00";
EXPECT_EQ(taosParseTime(t69, &time, strlen(t69), TSDB_TIME_PRECISION_MILLI, 0), -1);
char t70[] = "2017-12-31 9:0:0.123Z+12:00";
EXPECT_EQ(taosParseTime(t70, &time, strlen(t70), TSDB_TIME_PRECISION_MILLI, 0), -1);
}
/* test parse time profiling */
TEST(testCase, parse_time_profile) {
taos_options(TSDB_OPTION_TIMEZONE, "GMT-8");
char t1[] = "2018-1-8 1:1:1.952";
char t2[] = "2018-1-8T1:1:1.952+0800";
char t3[] = "2018-1-8 1:1:1.952+0800";
char t4[] = "2018-1-81:1:1.952+0800";
char t5[] = "2018-1-8 1:1:1.952";
char t6[] = "2018-1-8T1:1:1.952+08:00";
char t7[] = "2018-1-8 1:1:1.952+08:00";
char t8[] = "2018-1-81:1:1.952+08:00";
char t9[] = "2018-1-8 1:1:1.952";
char t10[] = "2018-1-8T1:1:1.952Z";
char t11[] = "2018-1-8 1:1:1.952z";
char t12[] = "2018-1-81:1:1.952Z";
struct timeval start, end;
int64_t time = 0, time1 = 0;
int32_t total_run = 100000000;
long total_time_us;
gettimeofday(&start, NULL);
for (int i = 0; i < total_run; ++i) {
taosParseTime(t1, &time, strlen(t1), TSDB_TIME_PRECISION_MILLI, 0);
}
gettimeofday(&end, NULL);
total_time_us = ((end.tv_sec - start.tv_sec)* 1000000) + (end.tv_usec - start.tv_usec);
printf("[t1] The elapsed time is %f seconds in %d run, average:%fns\n", total_time_us/1000000.0, total_run, 1000*(float)total_time_us/(float)total_run);
gettimeofday(&start, NULL);
for (int i = 0; i < total_run; ++i) {
taosParseTime(t2, &time, strlen(t2), TSDB_TIME_PRECISION_MILLI, 0);
}
gettimeofday(&end, NULL);
total_time_us = ((end.tv_sec - start.tv_sec)* 1000000) + (end.tv_usec - start.tv_usec);
printf("[t2] The elapsed time is %f seconds in %d run, average:%fns\n", total_time_us/1000000.0, total_run, 1000*(float)total_time_us/(float)total_run);
gettimeofday(&start, NULL);
for (int i = 0; i < total_run; ++i) {
taosParseTime(t3, &time, strlen(t3), TSDB_TIME_PRECISION_MILLI, 0);
}
gettimeofday(&end, NULL);
total_time_us = ((end.tv_sec - start.tv_sec)* 1000000) + (end.tv_usec - start.tv_usec);
printf("[t3] The elapsed time is %f seconds in %d run, average:%fns\n", total_time_us/1000000.0, total_run, 1000*(float)total_time_us/(float)total_run);
gettimeofday(&start, NULL);
for (int i = 0; i < total_run; ++i) {
taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI, 0);
}
gettimeofday(&end, NULL);
total_time_us = ((end.tv_sec - start.tv_sec)* 1000000) + (end.tv_usec - start.tv_usec);
printf("[t4] The elapsed time is %f seconds in %d run, average:%fns\n", total_time_us/1000000.0, total_run, 1000*(float)total_time_us/(float)total_run);
}
TEST(testCase, tvariant_convert) {
// 1. bool data to all other data types
tVariant t = {0};
_init_tvariant_bool(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0);
EXPECT_EQ(t.i64, 0);
_init_tvariant_bool(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0);
EXPECT_EQ(t.i64, 0);
_init_tvariant_bool(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0);
EXPECT_EQ(t.i64, 0);
_init_tvariant_bool(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0);
EXPECT_EQ(t.i64, 0);
_init_tvariant_bool(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0);
EXPECT_EQ(t.dKey, 0);
_init_tvariant_bool(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0);
EXPECT_EQ(t.dKey, 0);
_init_tvariant_bool(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0);
EXPECT_STREQ(t.pz, "FALSE");
tVariantDestroy(&t);
_init_tvariant_bool(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0);
EXPECT_STREQ(t.wpz, L"FALSE");
tVariantDestroy(&t);
// 2. tinyint to other data types
_init_tvariant_tinyint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0);
EXPECT_EQ(t.i64, 1);
_init_tvariant_tinyint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0);
EXPECT_EQ(t.i64, -27);
_init_tvariant_tinyint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0);
EXPECT_EQ(t.i64, -27);
_init_tvariant_tinyint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0);
EXPECT_EQ(t.i64, -27);
_init_tvariant_tinyint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0);
EXPECT_EQ(t.i64, -27);
_init_tvariant_tinyint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0);
EXPECT_EQ(t.dKey, -27);
_init_tvariant_tinyint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0);
EXPECT_EQ(t.dKey, -27);
_init_tvariant_tinyint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0);
EXPECT_STREQ(t.pz, "-27");
tVariantDestroy(&t);
_init_tvariant_tinyint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0);
EXPECT_STREQ(t.wpz, L"-27");
tVariantDestroy(&t);
// 3. int to other data
// types//////////////////////////////////////////////////////////////////
_init_tvariant_int(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0);
EXPECT_EQ(t.i64, 1);
_init_tvariant_int(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0);
_init_tvariant_int(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0);
_init_tvariant_int(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0);
EXPECT_EQ(t.i64, -23997659);
_init_tvariant_int(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0);
EXPECT_EQ(t.i64, -23997659);
_init_tvariant_int(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0);
EXPECT_EQ(t.dKey, -23997659);
_init_tvariant_int(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0);
EXPECT_EQ(t.dKey, -23997659);
_init_tvariant_int(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0);
EXPECT_STREQ(t.pz, "-23997659");
tVariantDestroy(&t);
_init_tvariant_int(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0);
EXPECT_STREQ(t.wpz, L"-23997659");
tVariantDestroy(&t);
// 4. bigint to other data
// type//////////////////////////////////////////////////////////////////////////////
_init_tvariant_bigint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0);
EXPECT_EQ(t.i64, 1);
_init_tvariant_bigint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0);
_init_tvariant_bigint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0);
_init_tvariant_bigint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0);
_init_tvariant_bigint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0);
EXPECT_EQ(t.i64, -3333333333333);
_init_tvariant_bigint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0);
EXPECT_EQ(t.dKey, -3333333333333);
_init_tvariant_bigint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0);
EXPECT_EQ(t.dKey, -3333333333333);
_init_tvariant_bigint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0);
EXPECT_STREQ(t.pz, "-3333333333333");
tVariantDestroy(&t);
_init_tvariant_bigint(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0);
EXPECT_STREQ(t.wpz, L"-3333333333333");
tVariantDestroy(&t);
// 5. float to other data
// types////////////////////////////////////////////////////////////////////////
_init_tvariant_float(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0);
EXPECT_EQ(t.i64, 1);
_init_tvariant_float(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0);
EXPECT_EQ(t.i64, -8991212199);
_init_tvariant_float(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0);
EXPECT_DOUBLE_EQ(t.dKey, -8991212199.8987885);
_init_tvariant_float(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0);
EXPECT_DOUBLE_EQ(t.dKey, -8991212199.8987885);
_init_tvariant_float(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0);
EXPECT_STREQ(t.pz, "-8991212199.898788");
tVariantDestroy(&t);
_init_tvariant_float(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0);
EXPECT_STREQ(t.wpz, L"-8991212199.898788");
tVariantDestroy(&t);
// 6. binary to other data types
// //////////////////////////////////////////////////////////////////
t.pz = "true";
t.nLen = strlen(t.pz);
t.nType = TSDB_DATA_TYPE_BINARY;
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0);
EXPECT_EQ(t.i64, 1);
_init_tvariant_binary(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), -1);
_init_tvariant_binary(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0);
EXPECT_EQ(t.i64, 200000);
_init_tvariant_binary(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0);
EXPECT_DOUBLE_EQ(t.dKey, 200000);
_init_tvariant_binary(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0);
EXPECT_DOUBLE_EQ(t.dKey, 200000);
_init_tvariant_binary(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0);
EXPECT_STREQ(t.pz, "2e5");
tVariantDestroy(&t);
_init_tvariant_binary(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0);
EXPECT_STREQ(t.wpz, L"2e5");
tVariantDestroy(&t);
// 7. nchar to other data types
// //////////////////////////////////////////////////////////////////
t.wpz = L"FALSE";
t.nLen = wcslen(t.wpz);
t.nType = TSDB_DATA_TYPE_NCHAR;
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0);
EXPECT_EQ(t.i64, 0);
_init_tvariant_nchar(&t);
EXPECT_LE(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0);
_init_tvariant_nchar(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0);
EXPECT_EQ(t.i64, -2000000);
_init_tvariant_nchar(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0);
EXPECT_DOUBLE_EQ(t.dKey, -2000000.8765);
_init_tvariant_nchar(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0);
EXPECT_DOUBLE_EQ(t.dKey, -2000000.8765);
_init_tvariant_nchar(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0);
EXPECT_STREQ(t.pz, "-2000000.8765");
tVariantDestroy(&t);
_init_tvariant_nchar(&t);
EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0);
EXPECT_STREQ(t.wpz, L"-2000000.8765");
tVariantDestroy(&t);
}
TEST(testCase, tGetToken_Test) {
char* s = ".123 ";
uint32_t type = 0;
int32_t len = tGetToken(s, &type);
EXPECT_EQ(type, TK_FLOAT);
EXPECT_EQ(len, strlen(s) - 1);
char s1[] = "1.123e10 ";
len = tGetToken(s1, &type);
EXPECT_EQ(type, TK_FLOAT);
EXPECT_EQ(len, strlen(s1) - 1);
char s4[] = "0xff ";
len = tGetToken(s4, &type);
EXPECT_EQ(type, TK_HEX);
EXPECT_EQ(len, strlen(s4) - 1);
// invalid data type
char s2[] = "e10 ";
len = tGetToken(s2, &type);
EXPECT_FALSE(type == TK_FLOAT);
char s3[] = "1.1.1.1";
len = tGetToken(s3, &type);
EXPECT_EQ(type, TK_IPTOKEN);
EXPECT_EQ(len, strlen(s3));
char s5[] = "0x ";
len = tGetToken(s5, &type);
EXPECT_FALSE(type == TK_HEX);
}
static SStrToken createStrToken(char* s) {
SStrToken t = {0};//.type = TK_STRING, .z = s, .n = strlen(s)};
t.type = TK_STRING;
t.z = s;
t.n = strlen(s);
return t;
}
TEST(testCase, isValidNumber_test) {
SStrToken t1 = createStrToken("123abc");
EXPECT_EQ(tGetNumericStringType(&t1), TK_ILLEGAL);
t1 = createStrToken("0xabc");
EXPECT_EQ(tGetNumericStringType(&t1), TK_HEX);
t1 = createStrToken("0b11101");
EXPECT_EQ(tGetNumericStringType(&t1), TK_BIN);
t1 = createStrToken(".134abc");
EXPECT_EQ(tGetNumericStringType(&t1), TK_ILLEGAL);
t1 = createStrToken("1e1 ");
EXPECT_EQ(tGetNumericStringType(&t1), TK_ILLEGAL);
t1 = createStrToken("1+2");
EXPECT_EQ(tGetNumericStringType(&t1), TK_ILLEGAL);
t1 = createStrToken("-0x123");
EXPECT_EQ(tGetNumericStringType(&t1), TK_HEX);
t1 = createStrToken("-1");
EXPECT_EQ(tGetNumericStringType(&t1), TK_INTEGER);
t1 = createStrToken("-0b1110");
EXPECT_EQ(tGetNumericStringType(&t1), TK_BIN);
t1 = createStrToken("-.234");
EXPECT_EQ(tGetNumericStringType(&t1), TK_FLOAT);
}
TEST(testCase, getTempFilePath_test) {
char path[4096] = {0};
memset(path, 1, 4096);
taosGetTmpfilePath("new_tmp", path);
printf("%s\n", path);
}
#pragma GCC diagnostic pop

10
include/client/taos.h
View File

@ -315,6 +315,16 @@ DLL_EXPORT TAOS_RES *tmq_create_stream(TAOS *taos, const char *streamName, const
typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB *tsub, TAOS_RES *res, void *param, int code);
#endif
typedef enum {
TSDB_SRV_STATUS_UNAVAILABLE = 0,
TSDB_SRV_STATUS_NETWORK_OK = 1,
TSDB_SRV_STATUS_SERVICE_OK = 2,
TSDB_SRV_STATUS_SERVICE_DEGRADED = 3,
TSDB_SRV_STATUS_EXTING = 4,
} TSDB_SERVER_STATUS;
DLL_EXPORT TSDB_SERVER_STATUS taos_check_server_status(const char *fqdn, int port, char *details, int maxlen);
#ifdef __cplusplus
}
#endif

2
include/common/tdatablock.h
View File

@ -211,7 +211,7 @@ size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock);
int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo);
int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst);
int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows);
int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, size_t existRows, uint32_t numOfRows);
int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows);
void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows);

1
include/common/tglobal.h
View File

@ -130,7 +130,6 @@ void taosCfgDynamicOptions(const char *option, const char *value);
void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary);
struct SConfig *taosGetCfg();
int32_t taosAddClientLogCfg(SConfig *pCfg);
#ifdef __cplusplus
}

96
include/common/tmsg.h
View File

@ -253,22 +253,16 @@ typedef struct {
SSubmitRspBlock failedBlocks[];
} SSubmitRsp;
#define SCHEMA_SMA_ON 0x1
#define SCHEMA_IDX_ON 0x2
typedef struct SSchema {
int8_t type;
int8_t index; // default is 0, not index created
int8_t flags;
col_id_t colId;
int32_t bytes;
char name[TSDB_COL_NAME_LEN];
} SSchema;
typedef struct {
int8_t type;
int8_t sma; // ETsdbBSmaType and default is TSDB_BSMA_TYPE_I
col_id_t colId;
int32_t bytes;
char name[TSDB_COL_NAME_LEN];
} SSchemaEx;
#define SSCHMEA_TYPE(s) ((s)->type)
#define SSCHMEA_SMA(s) ((s)->sma)
#define SSCHMEA_COLID(s) ((s)->colId)
@ -286,10 +280,14 @@ typedef struct {
int32_t numOfTags;
int32_t numOfSmas;
int32_t commentLen;
int32_t ast1Len;
int32_t ast2Len;
SArray* pColumns; // array of SField
SArray* pTags; // array of SField
SArray* pSmas; // array of SField
char* comment;
char* pAst1;
char* pAst2;
} SMCreateStbReq;
int32_t tSerializeSMCreateStbReq(void* buf, int32_t bufLen, SMCreateStbReq* pReq);
@ -515,7 +513,7 @@ typedef struct {
int8_t precision; // time resolution
int8_t compression;
int8_t replications;
int8_t quorum;
int8_t strict;
int8_t update;
int8_t cacheLastRow;
int8_t ignoreExist;
@ -537,7 +535,7 @@ typedef struct {
int32_t daysToKeep2;
int32_t fsyncPeriod;
int8_t walLevel;
int8_t quorum;
int8_t strict;
int8_t cacheLastRow;
int8_t replications;
} SAlterDbReq;
@ -610,11 +608,13 @@ typedef struct {
int8_t precision;
int8_t compression;
int8_t replications;
int8_t quorum;
int8_t strict;
int8_t update;
int8_t cacheLastRow;
int8_t streamMode;
int8_t singleSTable;
int32_t numOfRetensions;
SArray* pRetensions;
} SDbCfgRsp;
int32_t tSerializeSDbCfgRsp(void* buf, int32_t bufLen, const SDbCfgRsp* pRsp);
@ -645,10 +645,10 @@ void tFreeSUseDbBatchRsp(SUseDbBatchRsp* pRsp);
typedef struct {
char db[TSDB_DB_FNAME_LEN];
} SSyncDbReq, SCompactDbReq;
} SCompactDbReq;
int32_t tSerializeSSyncDbReq(void* buf, int32_t bufLen, SSyncDbReq* pReq);
int32_t tDeserializeSSyncDbReq(void* buf, int32_t bufLen, SSyncDbReq* pReq);
int32_t tSerializeSCompactDbReq(void* buf, int32_t bufLen, SCompactDbReq* pReq);
int32_t tDeserializeSCompactDbReq(void* buf, int32_t bufLen, SCompactDbReq* pReq);
typedef struct {
char name[TSDB_FUNC_NAME_LEN];
@ -729,7 +729,7 @@ typedef struct {
typedef struct {
int32_t vgId;
int8_t role;
int32_t syncState;
int64_t numOfTables;
int64_t numOfTimeSeries;
int64_t totalStorage;
@ -742,6 +742,10 @@ typedef struct {
int64_t numOfBatchInsertSuccessReqs;
} SVnodeLoad;
typedef struct {
int32_t syncState;
} SMnodeLoad;
typedef struct {
int32_t sver; // software version
int64_t dnodeVer; // dnode table version in sdb
@ -816,7 +820,7 @@ typedef struct {
int8_t walLevel;
int8_t precision;
int8_t compression;
int8_t quorum;
int8_t strict;
int8_t update;
int8_t cacheLastRow;
int8_t replica;
@ -825,7 +829,7 @@ typedef struct {
SReplica replicas[TSDB_MAX_REPLICA];
int32_t numOfRetensions;
SArray* pRetensions; // SRetention
} SCreateVnodeReq, SAlterVnodeReq;
} SCreateVnodeReq;
int32_t tSerializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq);
int32_t tDeserializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq);
@ -836,11 +840,36 @@ typedef struct {
int32_t dnodeId;
int64_t dbUid;
char db[TSDB_DB_FNAME_LEN];
} SDropVnodeReq, SSyncVnodeReq, SCompactVnodeReq;
} SDropVnodeReq;
int32_t tSerializeSDropVnodeReq(void* buf, int32_t bufLen, SDropVnodeReq* pReq);
int32_t tDeserializeSDropVnodeReq(void* buf, int32_t bufLen, SDropVnodeReq* pReq);
typedef struct {
int64_t dbUid;
char db[TSDB_DB_FNAME_LEN];
} SCompactVnodeReq;
int32_t tSerializeSCompactVnodeReq(void* buf, int32_t bufLen, SCompactVnodeReq* pReq);
int32_t tDeserializeSCompactVnodeReq(void* buf, int32_t bufLen, SCompactVnodeReq* pReq);
typedef struct {
int32_t vgVersion;
int32_t totalBlocks;
int32_t daysToKeep0;
int32_t daysToKeep1;
int32_t daysToKeep2;
int8_t walLevel;
int8_t strict;
int8_t cacheLastRow;
int8_t replica;
int8_t selfIndex;
SReplica replicas[TSDB_MAX_REPLICA];
} SAlterVnodeReq;
int32_t tSerializeSAlterVnodeReq(void* buf, int32_t bufLen, SAlterVnodeReq* pReq);
int32_t tDeserializeSAlterVnodeReq(void* buf, int32_t bufLen, SAlterVnodeReq* pReq);
typedef struct {
SMsgHead header;
char dbFName[TSDB_DB_FNAME_LEN];
@ -1077,10 +1106,12 @@ int32_t tSerializeSAuthReq(void* buf, int32_t bufLen, SAuthReq* pReq);
int32_t tDeserializeSAuthReq(void* buf, int32_t bufLen, SAuthReq* pReq);
typedef struct {
int8_t finished;
char name[TSDB_STEP_NAME_LEN];
char desc[TSDB_STEP_DESC_LEN];
} SStartupReq;
int32_t statusCode;
char details[1024];
} SServerStatusRsp;
int32_t tSerializeSServerStatusRsp(void* buf, int32_t bufLen, SServerStatusRsp* pRsp);
int32_t tDeserializeSServerStatusRsp(void* buf, int32_t bufLen, SServerStatusRsp* pRsp);
/**
* The layout of the query message payload is as following:
@ -1437,8 +1468,6 @@ typedef struct {
} SRSmaParam;
typedef struct SVCreateTbReq {
int64_t ver; // use a general definition
char* dbFName;
char* name;
uint32_t ttl;
uint32_t keep;
@ -1454,7 +1483,7 @@ typedef struct SVCreateTbReq {
tb_uid_t suid;
col_id_t nCols;
col_id_t nBSmaCols;
SSchemaEx* pSchema;
SSchema* pSchema;
col_id_t nTagCols;
SSchema* pTagSchema;
SRSmaParam* pRSmaParam;
@ -1466,7 +1495,7 @@ typedef struct SVCreateTbReq {
struct {
col_id_t nCols;
col_id_t nBSmaCols;
SSchemaEx* pSchema;
SSchema* pSchema;
SRSmaParam* pRSmaParam;
} ntbCfg;
};
@ -2031,16 +2060,13 @@ int32_t tDecodeSMqCMCommitOffsetReq(SCoder* decoder, SMqCMCommitOffsetReq* pReq)
typedef struct {
uint32_t nCols;
union {
SSchema* pSchema;
SSchemaEx* pSchemaEx;
};
SSchema* pSchema;
} SSchemaWrapper;
static FORCE_INLINE int32_t taosEncodeSSchema(void** buf, const SSchema* pSchema) {
int32_t tlen = 0;
tlen += taosEncodeFixedI8(buf, pSchema->type);
tlen += taosEncodeFixedI8(buf, pSchema->index);
tlen += taosEncodeFixedI8(buf, pSchema->flags);
tlen += taosEncodeFixedI32(buf, pSchema->bytes);
tlen += taosEncodeFixedI16(buf, pSchema->colId);
tlen += taosEncodeString(buf, pSchema->name);
@ -2049,7 +2075,7 @@ static FORCE_INLINE int32_t taosEncodeSSchema(void** buf, const SSchema* pSchema
static FORCE_INLINE void* taosDecodeSSchema(void* buf, SSchema* pSchema) {
buf = taosDecodeFixedI8(buf, &pSchema->type);
buf = taosDecodeFixedI8(buf, &pSchema->index);
buf = taosDecodeFixedI8(buf, &pSchema->flags);
buf = taosDecodeFixedI32(buf, &pSchema->bytes);
buf = taosDecodeFixedI16(buf, &pSchema->colId);
buf = taosDecodeStringTo(buf, pSchema->name);
@ -2058,7 +2084,7 @@ static FORCE_INLINE void* taosDecodeSSchema(void* buf, SSchema* pSchema) {
static FORCE_INLINE int32_t tEncodeSSchema(SCoder* pEncoder, const SSchema* pSchema) {
if (tEncodeI8(pEncoder, pSchema->type) < 0) return -1;
if (tEncodeI8(pEncoder, pSchema->index) < 0) return -1;
if (tEncodeI8(pEncoder, pSchema->flags) < 0) return -1;
if (tEncodeI32(pEncoder, pSchema->bytes) < 0) return -1;
if (tEncodeI16(pEncoder, pSchema->colId) < 0) return -1;
if (tEncodeCStr(pEncoder, pSchema->name) < 0) return -1;
@ -2067,7 +2093,7 @@ static FORCE_INLINE int32_t tEncodeSSchema(SCoder* pEncoder, const SSchema* pSch
static FORCE_INLINE int32_t tDecodeSSchema(SCoder* pDecoder, SSchema* pSchema) {
if (tDecodeI8(pDecoder, &pSchema->type) < 0) return -1;
if (tDecodeI8(pDecoder, &pSchema->index) < 0) return -1;
if (tDecodeI8(pDecoder, &pSchema->flags) < 0) return -1;
if (tDecodeI32(pDecoder, &pSchema->bytes) < 0) return -1;
if (tDecodeI16(pDecoder, &pSchema->colId) < 0) return -1;
if (tDecodeCStrTo(pDecoder, pSchema->name) < 0) return -1;

3
include/common/tmsgcb.h
View File

@ -44,6 +44,7 @@ typedef int32_t (*SendMnodeReqFp)(SMgmtWrapper* pWrapper, SRpcMsg* pReq);
typedef void (*SendRspFp)(SMgmtWrapper* pWrapper, const SRpcMsg* pRsp);
typedef void (*RegisterBrokenLinkArgFp)(SMgmtWrapper* pWrapper, SRpcMsg* pMsg);
typedef void (*ReleaseHandleFp)(SMgmtWrapper* pWrapper, void* handle, int8_t type);
typedef void (*ReportStartup)(SMgmtWrapper* pWrapper, const char* name, const char* desc);
typedef struct {
SMgmtWrapper* pWrapper;
@ -53,6 +54,7 @@ typedef struct {
SendRspFp sendRspFp;
RegisterBrokenLinkArgFp registerBrokenLinkArgFp;
ReleaseHandleFp releaseHandleFp;
ReportStartup reportStartupFp;
} SMsgCb;
void tmsgSetDefaultMsgCb(const SMsgCb* pMsgCb);
@ -62,6 +64,7 @@ int32_t tmsgSendReq(const SMsgCb* pMsgCb, const SEpSet* epSet, SRpcMsg* pReq);
void tmsgSendRsp(const SRpcMsg* pRsp);
void tmsgRegisterBrokenLinkArg(const SMsgCb* pMsgCb, SRpcMsg* pMsg);
void tmsgReleaseHandle(void* handle, int8_t type);
void tmsgReportStartup(const char* name, const char* desc);
#ifdef __cplusplus
}

10
include/common/tmsgdef.h
View File

@ -82,12 +82,9 @@ enum {
TD_DEF_MSG_TYPE(TDMT_DND_ALTER_BNODE, "dnode-alter-bnode", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_DND_DROP_BNODE, "dnode-drop-bnode", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_DND_CREATE_VNODE, "dnode-create-vnode", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_DND_ALTER_VNODE, "dnode-alter-vnode", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_DND_DROP_VNODE, "dnode-drop-vnode", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_DND_SYNC_VNODE, "dnode-sync-vnode", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_DND_COMPACT_VNODE, "dnode-compact-vnode", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_DND_CONFIG_DNODE, "dnode-config-dnode", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_DND_NETWORK_TEST, "dnode-nettest", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_DND_SERVER_STATUS, "dnode-server-status", NULL, NULL)
// Requests handled by MNODE
TD_NEW_MSG_SEG(TDMT_MND_MSG)
@ -206,6 +203,10 @@ enum {
TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_SYNC_VNODE, "vnode-sync-vnode", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_ALTER_VNODE, "vnode-alter-vnode", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_COMPACT_VNODE, "vnode-compact-vnode", NULL, NULL)
// Requests handled by QNODE
TD_NEW_MSG_SEG(TDMT_QND_MSG)
@ -228,6 +229,7 @@ enum {
TD_DEF_MSG_TYPE(TDMT_MON_SM_INFO, "monitor-sinfo", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MON_BM_INFO, "monitor-binfo", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MON_VM_LOAD, "monitor-vload", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MON_MM_LOAD, "monitor-mload", NULL, NULL)
#if defined(TD_MSG_NUMBER_)
TDMT_MAX

74
include/common/trow.h
View File

@ -308,21 +308,21 @@ static FORCE_INLINE int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx,
// use literal value directly and not use formula to simplify the codes
switch (nOffset) {
case 0:
// *pDestByte = ((*pDestByte) & 0x3F) | (valType << 6);
// set the value and clear other partitions for offset 0
*pDestByte = (valType << 6);
// *pDestByte |= (valType << 6);
break;
case 1:
// *pDestByte = ((*pDestByte) & 0xCF) | (valType << 4);
*pDestByte |= (valType << 4);
*pDestByte = ((*pDestByte) & 0xCF) | (valType << 4);
// *pDestByte |= (valType << 4);
break;
case 2:
// *pDestByte = ((*pDestByte) & 0xF3) | (valType << 2);
*pDestByte |= (valType << 2);
*pDestByte = ((*pDestByte) & 0xF3) | (valType << 2);
// *pDestByte |= (valType << 2);
break;
case 3:
// *pDestByte = ((*pDestByte) & 0xFC) | valType;
*pDestByte |= (valType);
*pDestByte = ((*pDestByte) & 0xFC) | valType;
// *pDestByte |= (valType);
break;
default:
TASSERT(0);
@ -417,37 +417,37 @@ static FORCE_INLINE int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, T
// use literal value directly and not use formula to simplify the codes
switch (nOffset) {
case 0:
// *pDestByte = ((*pDestByte) & 0x7F) | (valType << 7);
// set the value and clear other partitions for offset 0
*pDestByte = (valType << 7);
// *pDestByte |= (valType << 7);
break;
case 1:
// *pDestByte = ((*pDestByte) & 0xBF) | (valType << 6);
*pDestByte |= (valType << 6);
*pDestByte = ((*pDestByte) & 0xBF) | (valType << 6);
// *pDestByte |= (valType << 6);
break;
case 2:
// *pDestByte = ((*pDestByte) & 0xDF) | (valType << 5);
*pDestByte |= (valType << 5);
*pDestByte = ((*pDestByte) & 0xDF) | (valType << 5);
// *pDestByte |= (valType << 5);
break;
case 3:
// *pDestByte = ((*pDestByte) & 0xEF) | (valType << 4);
*pDestByte |= (valType << 4);
*pDestByte = ((*pDestByte) & 0xEF) | (valType << 4);
// *pDestByte |= (valType << 4);
break;
case 4:
// *pDestByte = ((*pDestByte) & 0xF7) | (valType << 3);
*pDestByte |= (valType << 3);
*pDestByte = ((*pDestByte) & 0xF7) | (valType << 3);
// *pDestByte |= (valType << 3);
break;
case 5:
// *pDestByte = ((*pDestByte) & 0xFB) | (valType << 2);
*pDestByte |= (valType << 2);
*pDestByte = ((*pDestByte) & 0xFB) | (valType << 2);
// *pDestByte |= (valType << 2);
break;
case 6:
// *pDestByte = ((*pDestByte) & 0xFD) | (valType << 1);
*pDestByte |= (valType << 1);
*pDestByte = ((*pDestByte) & 0xFD) | (valType << 1);
// *pDestByte |= (valType << 1);
break;
case 7:
// *pDestByte = ((*pDestByte) & 0xFE) | valType;
*pDestByte |= (valType);
*pDestByte = ((*pDestByte) & 0xFE) | valType;
// *pDestByte |= (valType);
break;
default:
TASSERT(0);
@ -651,11 +651,14 @@ static int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) {
TD_ROW_SET_INFO(pBuilder->pBuf, 0);
TD_ROW_SET_TYPE(pBuilder->pBuf, pBuilder->rowType);
TASSERT(pBuilder->nBitmaps > 0 && pBuilder->flen > 0);
uint32_t len = 0;
switch (pBuilder->rowType) {
case TD_ROW_TP:
#ifdef TD_SUPPORT_BITMAP
pBuilder->pBitmap = tdGetBitmapAddrTp(pBuilder->pBuf, pBuilder->flen);
memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBitmaps);
#endif
// the primary TS key is stored separatedly
len = TD_ROW_HEAD_LEN + pBuilder->flen - sizeof(TSKEY) + pBuilder->nBitmaps;
@ -665,6 +668,7 @@ static int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) {
case TD_ROW_KV:
#ifdef TD_SUPPORT_BITMAP
pBuilder->pBitmap = tdGetBitmapAddrKv(pBuilder->pBuf, pBuilder->nBoundCols);
memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBitmaps);
#endif
len = TD_ROW_HEAD_LEN + TD_ROW_NCOLS_LEN + (pBuilder->nBoundCols - 1) * sizeof(SKvRowIdx) +
pBuilder->nBoundBitmaps; // add
@ -1163,6 +1167,18 @@ static FORCE_INLINE int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, in
return TSDB_CODE_SUCCESS;
}
/**
* @brief
*
* @param pRow
* @param colId
* @param colType
* @param flen
* @param offset
* @param colIdx start from 0
* @param pVal
* @return FORCE_INLINE
*/
static FORCE_INLINE bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t flen, uint32_t offset,
col_id_t colIdx, SCellVal *pVal) {
if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
@ -1170,10 +1186,20 @@ static FORCE_INLINE bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t
return true;
}
void *pBitmap = tdGetBitmapAddrTp(pRow, flen);
tdGetTpRowValOfCol(pVal, pRow, pBitmap, colType, offset - sizeof(TSKEY), colIdx - 1);
tdGetTpRowValOfCol(pVal, pRow, pBitmap, colType, offset - sizeof(TSKEY), colIdx);
return true;
}
/**
* @brief
*
* @param pRow
* @param colId
* @param offset
* @param colIdx start from 0
* @param pVal
* @return FORCE_INLINE
*/
static FORCE_INLINE bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, uint32_t offset, col_id_t colIdx,
SCellVal *pVal) {
if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
@ -1181,7 +1207,7 @@ static FORCE_INLINE bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, uint32_t o
return true;
}
void *pBitmap = tdGetBitmapAddrKv(pRow, tdRowGetNCols(pRow));
tdGetKvRowValOfCol(pVal, pRow, pBitmap, offset, colIdx - 1);
tdGetKvRowValOfCol(pVal, pRow, pBitmap, offset, colIdx);
return true;
}

299
include/common/ttokendef.h
View File

@ -86,155 +86,156 @@
#define TK_SINGLE_STABLE 68
#define TK_STREAM_MODE 69
#define TK_RETENTIONS 70
#define TK_NK_COMMA 71
#define TK_NK_COLON 72
#define TK_TABLE 73
#define TK_NK_LP 74
#define TK_NK_RP 75
#define TK_STABLE 76
#define TK_ADD 77
#define TK_COLUMN 78
#define TK_MODIFY 79
#define TK_RENAME 80
#define TK_TAG 81
#define TK_SET 82
#define TK_NK_EQ 83
#define TK_USING 84
#define TK_TAGS 85
#define TK_NK_DOT 86
#define TK_COMMENT 87
#define TK_BOOL 88
#define TK_TINYINT 89
#define TK_SMALLINT 90
#define TK_INT 91
#define TK_INTEGER 92
#define TK_BIGINT 93
#define TK_FLOAT 94
#define TK_DOUBLE 95
#define TK_BINARY 96
#define TK_TIMESTAMP 97
#define TK_NCHAR 98
#define TK_UNSIGNED 99
#define TK_JSON 100
#define TK_VARCHAR 101
#define TK_MEDIUMBLOB 102
#define TK_BLOB 103
#define TK_VARBINARY 104
#define TK_DECIMAL 105
#define TK_SMA 106
#define TK_ROLLUP 107
#define TK_FILE_FACTOR 108
#define TK_NK_FLOAT 109
#define TK_DELAY 110
#define TK_SHOW 111
#define TK_DATABASES 112
#define TK_TABLES 113
#define TK_STABLES 114
#define TK_MNODES 115
#define TK_MODULES 116
#define TK_QNODES 117
#define TK_FUNCTIONS 118
#define TK_INDEXES 119
#define TK_FROM 120
#define TK_ACCOUNTS 121
#define TK_APPS 122
#define TK_CONNECTIONS 123
#define TK_LICENCE 124
#define TK_GRANTS 125
#define TK_QUERIES 126
#define TK_SCORES 127
#define TK_TOPICS 128
#define TK_VARIABLES 129
#define TK_BNODES 130
#define TK_SNODES 131
#define TK_LIKE 132
#define TK_INDEX 133
#define TK_FULLTEXT 134
#define TK_FUNCTION 135
#define TK_INTERVAL 136
#define TK_TOPIC 137
#define TK_AS 138
#define TK_DESC 139
#define TK_DESCRIBE 140
#define TK_RESET 141
#define TK_QUERY 142
#define TK_EXPLAIN 143
#define TK_ANALYZE 144
#define TK_VERBOSE 145
#define TK_NK_BOOL 146
#define TK_RATIO 147
#define TK_COMPACT 148
#define TK_VNODES 149
#define TK_IN 150
#define TK_OUTPUTTYPE 151
#define TK_AGGREGATE 152
#define TK_BUFSIZE 153
#define TK_STREAM 154
#define TK_INTO 155
#define TK_TRIGGER 156
#define TK_AT_ONCE 157
#define TK_WINDOW_CLOSE 158
#define TK_WATERMARK 159
#define TK_KILL 160
#define TK_CONNECTION 161
#define TK_MERGE 162
#define TK_VGROUP 163
#define TK_REDISTRIBUTE 164
#define TK_SPLIT 165
#define TK_SYNCDB 166
#define TK_NULL 167
#define TK_NK_QUESTION 168
#define TK_NK_ARROW 169
#define TK_ROWTS 170
#define TK_TBNAME 171
#define TK_QSTARTTS 172
#define TK_QENDTS 173
#define TK_WSTARTTS 174
#define TK_WENDTS 175
#define TK_WDURATION 176
#define TK_CAST 177
#define TK_NOW 178
#define TK_TODAY 179
#define TK_TIMEZONE 180
#define TK_COUNT 181
#define TK_FIRST 182
#define TK_LAST 183
#define TK_LAST_ROW 184
#define TK_BETWEEN 185
#define TK_IS 186
#define TK_NK_LT 187
#define TK_NK_GT 188
#define TK_NK_LE 189
#define TK_NK_GE 190
#define TK_NK_NE 191
#define TK_MATCH 192
#define TK_NMATCH 193
#define TK_CONTAINS 194
#define TK_JOIN 195
#define TK_INNER 196
#define TK_SELECT 197
#define TK_DISTINCT 198
#define TK_WHERE 199
#define TK_PARTITION 200
#define TK_BY 201
#define TK_SESSION 202
#define TK_STATE_WINDOW 203
#define TK_SLIDING 204
#define TK_FILL 205
#define TK_VALUE 206
#define TK_NONE 207
#define TK_PREV 208
#define TK_LINEAR 209
#define TK_NEXT 210
#define TK_GROUP 211
#define TK_HAVING 212
#define TK_ORDER 213
#define TK_SLIMIT 214
#define TK_SOFFSET 215
#define TK_LIMIT 216
#define TK_OFFSET 217
#define TK_ASC 218
#define TK_NULLS 219
#define TK_STRICT 71
#define TK_NK_COMMA 72
#define TK_NK_COLON 73
#define TK_TABLE 74
#define TK_NK_LP 75
#define TK_NK_RP 76
#define TK_STABLE 77
#define TK_ADD 78
#define TK_COLUMN 79
#define TK_MODIFY 80
#define TK_RENAME 81
#define TK_TAG 82
#define TK_SET 83
#define TK_NK_EQ 84
#define TK_USING 85
#define TK_TAGS 86
#define TK_NK_DOT 87
#define TK_COMMENT 88
#define TK_BOOL 89
#define TK_TINYINT 90
#define TK_SMALLINT 91
#define TK_INT 92
#define TK_INTEGER 93
#define TK_BIGINT 94
#define TK_FLOAT 95
#define TK_DOUBLE 96
#define TK_BINARY 97
#define TK_TIMESTAMP 98
#define TK_NCHAR 99
#define TK_UNSIGNED 100
#define TK_JSON 101
#define TK_VARCHAR 102
#define TK_MEDIUMBLOB 103
#define TK_BLOB 104
#define TK_VARBINARY 105
#define TK_DECIMAL 106
#define TK_SMA 107
#define TK_ROLLUP 108
#define TK_FILE_FACTOR 109
#define TK_NK_FLOAT 110
#define TK_DELAY 111
#define TK_SHOW 112
#define TK_DATABASES 113
#define TK_TABLES 114
#define TK_STABLES 115
#define TK_MNODES 116
#define TK_MODULES 117
#define TK_QNODES 118
#define TK_FUNCTIONS 119
#define TK_INDEXES 120
#define TK_FROM 121
#define TK_ACCOUNTS 122
#define TK_APPS 123
#define TK_CONNECTIONS 124
#define TK_LICENCE 125
#define TK_GRANTS 126
#define TK_QUERIES 127
#define TK_SCORES 128
#define TK_TOPICS 129
#define TK_VARIABLES 130
#define TK_BNODES 131
#define TK_SNODES 132
#define TK_LIKE 133
#define TK_INDEX 134
#define TK_FULLTEXT 135
#define TK_FUNCTION 136
#define TK_INTERVAL 137
#define TK_TOPIC 138
#define TK_AS 139
#define TK_DESC 140
#define TK_DESCRIBE 141
#define TK_RESET 142
#define TK_QUERY 143
#define TK_EXPLAIN 144
#define TK_ANALYZE 145
#define TK_VERBOSE 146
#define TK_NK_BOOL 147
#define TK_RATIO 148
#define TK_COMPACT 149
#define TK_VNODES 150
#define TK_IN 151
#define TK_OUTPUTTYPE 152
#define TK_AGGREGATE 153
#define TK_BUFSIZE 154
#define TK_STREAM 155
#define TK_INTO 156
#define TK_TRIGGER 157
#define TK_AT_ONCE 158
#define TK_WINDOW_CLOSE 159
#define TK_WATERMARK 160
#define TK_KILL 161
#define TK_CONNECTION 162
#define TK_MERGE 163
#define TK_VGROUP 164
#define TK_REDISTRIBUTE 165
#define TK_SPLIT 166
#define TK_SYNCDB 167
#define TK_NULL 168
#define TK_NK_QUESTION 169
#define TK_NK_ARROW 170
#define TK_ROWTS 171
#define TK_TBNAME 172
#define TK_QSTARTTS 173
#define TK_QENDTS 174
#define TK_WSTARTTS 175
#define TK_WENDTS 176
#define TK_WDURATION 177
#define TK_CAST 178
#define TK_NOW 179
#define TK_TODAY 180
#define TK_TIMEZONE 181
#define TK_COUNT 182
#define TK_FIRST 183
#define TK_LAST 184
#define TK_LAST_ROW 185
#define TK_BETWEEN 186
#define TK_IS 187
#define TK_NK_LT 188
#define TK_NK_GT 189
#define TK_NK_LE 190
#define TK_NK_GE 191
#define TK_NK_NE 192
#define TK_MATCH 193
#define TK_NMATCH 194
#define TK_CONTAINS 195
#define TK_JOIN 196
#define TK_INNER 197
#define TK_SELECT 198
#define TK_DISTINCT 199
#define TK_WHERE 200
#define TK_PARTITION 201
#define TK_BY 202
#define TK_SESSION 203
#define TK_STATE_WINDOW 204
#define TK_SLIDING 205
#define TK_FILL 206
#define TK_VALUE 207
#define TK_NONE 208
#define TK_PREV 209
#define TK_LINEAR 210
#define TK_NEXT 211
#define TK_GROUP 212
#define TK_HAVING 213
#define TK_ORDER 214
#define TK_SLIMIT 215
#define TK_SOFFSET 216
#define TK_LIMIT 217
#define TK_OFFSET 218
#define TK_ASC 219
#define TK_NULLS 220
#define TK_NK_SPACE 300
#define TK_NK_COMMENT 301

1
include/dnode/mnode/mnode.h
View File

@ -80,6 +80,7 @@ void mndStop(SMnode *pMnode);
* @return int32_t 0 for success, -1 for failure.
*/
int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pCluster, SMonVgroupInfo *pVgroup, SMonGrantInfo *pGrant);
int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad);
/**
* @brief Process the read, write, sync request.

26
include/libs/function/function.h
View File

@ -295,19 +295,19 @@ typedef struct SPoint {
void * val;
} SPoint;
void taosFillSetStartInfo(struct SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey);
void taosResetFillInfo(struct SFillInfo* pFillInfo, TSKEY startTimestamp);
void taosFillSetInputDataBlock(struct SFillInfo* pFillInfo, const struct SSDataBlock* pInput);
struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, const int64_t* fillVal);
bool taosFillHasMoreResults(struct SFillInfo* pFillInfo);
struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols,
int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType,
struct SFillColInfo* pFillCol, const char* id);
void* taosDestroyFillInfo(struct SFillInfo *pFillInfo);
int64_t taosFillResultDataBlock(struct SFillInfo* pFillInfo, void** output, int32_t capacity);
int64_t getFillInfoStart(struct SFillInfo *pFillInfo);
//void taosFillSetStartInfo(struct SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey);
//void taosResetFillInfo(struct SFillInfo* pFillInfo, TSKEY startTimestamp);
//void taosFillSetInputDataBlock(struct SFillInfo* pFillInfo, const struct SSDataBlock* pInput);
//struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, const SValueNode* val);
//bool taosFillHasMoreResults(struct SFillInfo* pFillInfo);
//
//struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols,
// SInterval* pInterval, int32_t fillType,
// struct SFillColInfo* pCol, const char* id);
//
//void* taosDestroyFillInfo(struct SFillInfo *pFillInfo);
//int64_t taosFillResultDataBlock(struct SFillInfo* pFillInfo, void** output, int32_t capacity);
//int64_t getFillInfoStart(struct SFillInfo *pFillInfo);
int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2, int32_t inputType);

8
include/libs/monitor/monitor.h
View File

@ -202,6 +202,14 @@ int32_t tSerializeSMonVloadInfo(void *buf, int32_t bufLen, SMonVloadInfo *pInfo)
int32_t tDeserializeSMonVloadInfo(void *buf, int32_t bufLen, SMonVloadInfo *pInfo);
void tFreeSMonVloadInfo(SMonVloadInfo *pInfo);
typedef struct {
int8_t isMnode;
SMnodeLoad load;
} SMonMloadInfo;
int32_t tSerializeSMonMloadInfo(void *buf, int32_t bufLen, SMonMloadInfo *pInfo);
int32_t tDeserializeSMonMloadInfo(void *buf, int32_t bufLen, SMonMloadInfo *pInfo);
typedef struct {
const char *server;
uint16_t port;

1
include/libs/nodes/cmdnodes.h
View File

@ -47,6 +47,7 @@ typedef struct SDatabaseOptions {
SValueNode* pNumOfVgroups;
SValueNode* pSingleStable;
SValueNode* pStreamMode;
SValueNode* pStrict;
SNodeList* pRetentions;
} SDatabaseOptions;

2
include/libs/sync/sync.h
View File

@ -172,6 +172,8 @@ int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
extern int32_t sDebugFlag;
const char *syncStr(ESyncState state);
#ifdef __cplusplus
}
#endif

13
include/os/osThread.h
View File

@ -22,6 +22,10 @@
extern "C" {
#endif
#ifndef __USE_XOPEN2K
typedef pthread_mutex_t pthread_spinlock_t;
#endif
typedef pthread_t TdThread;
typedef pthread_spinlock_t TdThreadSpinlock;
typedef pthread_mutex_t TdThreadMutex;
@ -33,8 +37,6 @@ typedef pthread_rwlockattr_t TdThreadRwlockAttr;
typedef pthread_cond_t TdThreadCond;
typedef pthread_condattr_t TdThreadCondAttr;
typedef pthread_key_t TdThreadKey;
typedef pthread_barrier_t TdThreadBarrier;
typedef pthread_barrierattr_t TdThreadBarrierAttr;
#define taosThreadCleanupPush pthread_cleanup_push
#define taosThreadCleanupPop pthread_cleanup_pop
@ -156,13 +158,6 @@ int32_t taosThreadAttrSetSchedParam(TdThreadAttr * attr, const struct sched_para
int32_t taosThreadAttrSetSchedPolicy(TdThreadAttr * attr, int32_t policy);
int32_t taosThreadAttrSetScope(TdThreadAttr * attr, int32_t contentionscope);
int32_t taosThreadAttrSetStackSize(TdThreadAttr * attr, size_t stacksize);
int32_t taosThreadBarrierDestroy(TdThreadBarrier * barrier);
int32_t taosThreadBarrierInit(TdThreadBarrier * barrier, const TdThreadBarrierAttr * attr, uint32_t count);
int32_t taosThreadBarrierWait(TdThreadBarrier * barrier);
int32_t taosThreadBarrierAttrDestroy(TdThreadBarrierAttr * attr);
int32_t taosThreadBarrierAttrGetPshared(const TdThreadBarrierAttr * attr, int32_t *pshared);
int32_t taosThreadBarrierAttrInit(TdThreadBarrierAttr * attr);
int32_t taosThreadBarrierAttrSetPshared(TdThreadBarrierAttr * attr, int32_t pshared);
int32_t taosThreadCancel(TdThread thread);
int32_t taosThreadCondDestroy(TdThreadCond * cond);
int32_t taosThreadCondInit(TdThreadCond * cond, const TdThreadCondAttr * attr);

1
include/util/taoserror.h
View File

@ -618,6 +618,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_FUNC_FUNTION_ERROR TAOS_DEF_ERROR_CODE(0, 0x2800)
#define TSDB_CODE_FUNC_FUNTION_PARA_NUM TAOS_DEF_ERROR_CODE(0, 0x2801)
#define TSDB_CODE_FUNC_FUNTION_PARA_TYPE TAOS_DEF_ERROR_CODE(0, 0x2802)
#define TSDB_CODE_FUNC_FUNTION_PARA_VALUE TAOS_DEF_ERROR_CODE(0, 0x2803)
#ifdef __cplusplus
}

1
include/util/tconfig.h
View File

@ -32,6 +32,7 @@ typedef enum {
CFG_STYPE_ENV_VAR,
CFG_STYPE_APOLLO_URL,
CFG_STYPE_ARG_LIST,
CFG_STYPE_TAOS_OPTIONS
} ECfgSrcType;
typedef enum {

162
include/util/tdef.h
View File

@ -199,7 +199,7 @@ typedef enum EOperatorType {
} EOperatorType;
typedef enum ELogicConditionType {
LOGIC_COND_TYPE_AND,
LOGIC_COND_TYPE_AND = 1,
LOGIC_COND_TYPE_OR,
LOGIC_COND_TYPE_NOT,
} ELogicConditionType;
@ -299,16 +299,6 @@ typedef enum ELogicConditionType {
#define TSDB_DNODE_CONFIG_LEN 128
#define TSDB_DNODE_VALUE_LEN 256
#define TSDB_MQTT_HOSTNAME_LEN 64
#define TSDB_MQTT_PORT_LEN 8
#define TSDB_MQTT_USER_LEN 24
#define TSDB_MQTT_PASS_LEN 24
#define TSDB_MQTT_TOPIC_LEN 64
#define TSDB_MQTT_CLIENT_ID_LEN 32
#define TSDB_DB_TYPE_DEFAULT 0
#define TSDB_DB_TYPE_TOPIC 1
#define TSDB_DEFAULT_PKT_SIZE 65480 // same as RPC_MAX_UDP_SIZE
#define TSDB_PAYLOAD_SIZE TSDB_DEFAULT_PKT_SIZE
@ -317,9 +307,6 @@ typedef enum ELogicConditionType {
#define TSDB_CQ_SQL_SIZE 1024
#define TSDB_MIN_VNODES 16
#define TSDB_MAX_VNODES 512
#define TSDB_MIN_VNODES_PER_DB 1
#define TSDB_MAX_VNODES_PER_DB 4096
#define TSDB_DEFAULT_VN_PER_DB 2
#define TSDB_DNODE_ROLE_ANY 0
#define TSDB_DNODE_ROLE_MGMT 1
@ -333,104 +320,81 @@ typedef enum ELogicConditionType {
#define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta
#define TSDB_MIN_VNODES_PER_DB 1
#define TSDB_MAX_VNODES_PER_DB 4096
#define TSDB_DEFAULT_VN_PER_DB 2
#define TSDB_MIN_CACHE_BLOCK_SIZE 1
#define TSDB_MAX_CACHE_BLOCK_SIZE 128 // 128MB for each vnode
#define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16
#define TSDB_MIN_TOTAL_BLOCKS 3
#define TSDB_MAX_TOTAL_BLOCKS 10000
#define TSDB_DEFAULT_TOTAL_BLOCKS 6
#define TSDB_MIN_DAYS_PER_FILE 60 // unit minute
#define TSDB_MAX_DAYS_PER_FILE (3650 * 1440)
#define TSDB_DEFAULT_DAYS_PER_FILE (10 * 1440)
#define TSDB_MIN_KEEP (1 * 1440) // data in db to be reserved. unit minute
#define TSDB_MAX_KEEP (365000 * 1440) // data in db to be reserved.
#define TSDB_DEFAULT_KEEP (3650 * 1440) // ten years
#define TSDB_MIN_MIN_ROW_FBLOCK 10
#define TSDB_MAX_MIN_ROW_FBLOCK 1000
#define TSDB_DEFAULT_MIN_ROW_FBLOCK 100
#define TSDB_MIN_MAX_ROW_FBLOCK 200
#define TSDB_MAX_MAX_ROW_FBLOCK 10000
#define TSDB_DEFAULT_MAX_ROW_FBLOCK 4096
#define TSDB_MIN_COMMIT_TIME 30
#define TSDB_MAX_COMMIT_TIME 40960
#define TSDB_DEFAULT_COMMIT_TIME 3600
#define TSDB_MIN_FSYNC_PERIOD 0
#define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond
#define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second
#define TSDB_MIN_WAL_LEVEL 1
#define TSDB_MAX_WAL_LEVEL 2
#define TSDB_DEFAULT_WAL_LEVEL 1
#define TSDB_MIN_PRECISION TSDB_TIME_PRECISION_MILLI
#define TSDB_MAX_PRECISION TSDB_TIME_PRECISION_NANO
#define TSDB_DEFAULT_PRECISION TSDB_TIME_PRECISION_MILLI
#define TSDB_MIN_COMP_LEVEL 0
#define TSDB_MAX_COMP_LEVEL 2
#define TSDB_DEFAULT_COMP_LEVEL 2
#define TSDB_MIN_DB_REPLICA_OPTION 1
#define TSDB_MAX_DB_REPLICA_OPTION 3
#define TSDB_DEFAULT_DB_REPLICA_OPTION 1
#define TSDB_MIN_DB_QUORUM_OPTION 1
#define TSDB_MAX_DB_QUORUM_OPTION 2
#define TSDB_DEFAULT_DB_QUORUM_OPTION 1
#define TSDB_MIN_DB_TTL_OPTION 1
#define TSDB_DEFAULT_DB_TTL_OPTION 0
#define TSDB_MIN_DB_SINGLE_STABLE_OPTION 0
#define TSDB_MAX_DB_SINGLE_STABLE_OPTION 1
#define TSDB_DEFAULT_DB_SINGLE_STABLE_OPTION 0
#define TSDB_MIN_DB_STREAM_MODE_OPTION 0
#define TSDB_MAX_DB_STREAM_MODE_OPTION 1
#define TSDB_DEFAULT_DB_STREAM_MODE_OPTION 0
#define TSDB_MAX_JOIN_TABLE_NUM 10
#define TSDB_MAX_UNION_CLAUSE 5
#define TSDB_MIN_TOTAL_BLOCKS 3
#define TSDB_MAX_TOTAL_BLOCKS 10000
#define TSDB_DEFAULT_TOTAL_BLOCKS 6
#define TSDB_MIN_DAYS_PER_FILE 60 // unit minute
#define TSDB_MAX_DAYS_PER_FILE (3650 * 1440)
#define TSDB_DEFAULT_DAYS_PER_FILE (10 * 1440)
#define TSDB_MIN_KEEP (1 * 1440) // data in db to be reserved. unit minute
#define TSDB_MAX_KEEP (365000 * 1440) // data in db to be reserved.
#define TSDB_DEFAULT_KEEP (3650 * 1440) // ten years
#define TSDB_MIN_MINROWS_FBLOCK 10
#define TSDB_MAX_MINROWS_FBLOCK 1000
#define TSDB_DEFAULT_MINROWS_FBLOCK 100
#define TSDB_MIN_MAXROWS_FBLOCK 200
#define TSDB_MAX_MAXROWS_FBLOCK 10000
#define TSDB_DEFAULT_MAXROWS_FBLOCK 4096
#define TSDB_MIN_COMMIT_TIME 30
#define TSDB_MAX_COMMIT_TIME 40960
#define TSDB_DEFAULT_COMMIT_TIME 3600
#define TSDB_MIN_FSYNC_PERIOD 0
#define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond
#define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second
#define TSDB_MIN_DB_TTL 1
#define TSDB_DEFAULT_DB_TTL 1
#define TSDB_MIN_WAL_LEVEL 1
#define TSDB_MAX_WAL_LEVEL 2
#define TSDB_DEFAULT_WAL_LEVEL 1
#define TSDB_MIN_PRECISION TSDB_TIME_PRECISION_MILLI
#define TSDB_MAX_PRECISION TSDB_TIME_PRECISION_NANO
#define TSDB_DEFAULT_PRECISION TSDB_TIME_PRECISION_MILLI
#define TSDB_MIN_COMP_LEVEL 0
#define TSDB_MAX_COMP_LEVEL 2
#define TSDB_DEFAULT_COMP_LEVEL 2
#define TSDB_MIN_DB_REPLICA 1
#define TSDB_MAX_DB_REPLICA 3
#define TSDB_DEFAULT_DB_REPLICA 1
#define TSDB_DB_STRICT_OFF 0
#define TSDB_DB_STRICT_ON 1
#define TSDB_DEFAULT_DB_STRICT 0
#define TSDB_MIN_DB_UPDATE 0
#define TSDB_MAX_DB_UPDATE 2
#define TSDB_DEFAULT_DB_UPDATE_OPTION 0
#define TSDB_MIN_DB_CACHE_LAST_ROW 0
#define TSDB_MAX_DB_CACHE_LAST_ROW 3
#define TSDB_DEFAULT_CACHE_LAST_ROW 0
#define TSDB_MIN_DB_STREAM_MODE 0
#define TSDB_MAX_DB_STREAM_MODE 1
#define TSDB_DEFAULT_DB_STREAM_MODE 0
#define TSDB_DEFAULT_DB_UPDATE 0
#define TSDB_MIN_DB_CACHE_LAST_ROW 0
#define TSDB_MAX_DB_CACHE_LAST_ROW 3
#define TSDB_DEFAULT_CACHE_LAST_ROW 0
#define TSDB_DB_STREAM_MODE_OFF 0
#define TSDB_DB_STREAM_MODE_ON 1
#define TSDB_DEFAULT_DB_STREAM_MODE 0
#define TSDB_DB_SINGLE_STABLE_ON 0
#define TSDB_DB_SINGLE_STABLE_OFF 1
#define TSDB_DEFAULT_DB_SINGLE_STABLE 0
#define TSDB_MIN_DB_FILE_FACTOR 0
#define TSDB_MAX_DB_FILE_FACTOR 1
#define TSDB_DEFAULT_DB_FILE_FACTOR 0.1
#define TSDB_MIN_DB_DELAY 1
#define TSDB_MAX_DB_DELAY 10
#define TSDB_DEFAULT_DB_DELAY 2
#define TSDB_DEFAULT_EXPLAIN_VERBOSE false
#define TSDB_MIN_EXPLAIN_RATIO 0
#define TSDB_MAX_EXPLAIN_RATIO 1
#define TSDB_DEFAULT_EXPLAIN_RATIO 0.001
#define TSDB_EXPLAIN_RESULT_ROW_SIZE 1024
#define TSDB_EXPLAIN_RESULT_COLUMN_NAME "QUERY PLAN"
#define TSDB_MIN_DB_DELAY 1
#define TSDB_MAX_DB_DELAY 10
#define TSDB_DEFAULT_DB_DELAY 2
#define TSDB_MIN_EXPLAIN_RATIO 0
#define TSDB_MAX_EXPLAIN_RATIO 1
#define TSDB_DEFAULT_EXPLAIN_RATIO 0.001
#define TSDB_MAX_JOIN_TABLE_NUM 10
#define TSDB_MAX_UNION_CLAUSE 5
#define TSDB_DEFAULT_EXPLAIN_VERBOSE false
#define TSDB_EXPLAIN_RESULT_ROW_SIZE 1024
#define TSDB_EXPLAIN_RESULT_COLUMN_NAME "QUERY PLAN"
#define TSDB_MAX_FIELD_LEN 16384
#define TSDB_MAX_BINARY_LEN (TSDB_MAX_FIELD_LEN - TSDB_KEYSIZE) // keep 16384
#define TSDB_MAX_NCHAR_LEN (TSDB_MAX_FIELD_LEN - TSDB_KEYSIZE) // keep 16384

178
source/client/src/clientEnv.c
View File

@ -307,164 +307,50 @@ int taos_init() {
}
int taos_options_imp(TSDB_OPTION option, const char *str) {
#if 0
SGlobalCfg *cfg = NULL;
if (option != TSDB_OPTION_CONFIGDIR) {
taos_init(); // initialize global config
} else {
tstrncpy(configDir, str, PATH_MAX);
tscInfo("set cfg:%s to %s", configDir, str);
return 0;
}
SConfig *pCfg = taosGetCfg();
SConfigItem *pItem = NULL;
switch (option) {
case TSDB_OPTION_CONFIGDIR:
cfg = taosGetConfigOption("configDir");
assert(cfg != NULL);
if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) {
tstrncpy(configDir, str, TSDB_FILENAME_LEN);
cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
tscInfo("set config file directory:%s", str);
} else {
tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str,
tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr);
}
pItem = cfgGetItem(pCfg, "configDir");
break;
case TSDB_OPTION_SHELL_ACTIVITY_TIMER:
cfg = taosGetConfigOption("shellActivityTimer");
assert(cfg != NULL);
if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) {
tsShellActivityTimer = atoi(str);
if (tsShellActivityTimer < 1) tsShellActivityTimer = 1;
if (tsShellActivityTimer > 3600) tsShellActivityTimer = 3600;
cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
tscInfo("set shellActivityTimer:%d", tsShellActivityTimer);
} else {
tscWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, str,
tsCfgStatusStr[cfg->cfgStatus], *(int32_t *)cfg->ptr);
}
pItem = cfgGetItem(pCfg, "shellActivityTimer");
break;
case TSDB_OPTION_LOCALE: { // set locale
cfg = taosGetConfigOption("locale");
assert(cfg != NULL);
size_t len = strlen(str);
if (len == 0 || len > TD_LOCALE_LEN) {
tscInfo("Invalid locale:%s, use default", str);
return -1;
}
if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) {
char sep = '.';
if (strlen(tsLocale) == 0) { // locale does not set yet
char *defaultLocale = setlocale(LC_CTYPE, "");
// The locale of the current OS does not be set correctly, so the default locale cannot be acquired.
// The launch of current system will abort soon.
if (defaultLocale == NULL) {
tscError("failed to get default locale, please set the correct locale in current OS");
return -1;
}
tstrncpy(tsLocale, defaultLocale, TD_LOCALE_LEN);
}
// set the user specified locale
char *locale = setlocale(LC_CTYPE, str);
if (locale != NULL) { // failed to set the user specified locale
tscInfo("locale set, prev locale:%s, new locale:%s", tsLocale, locale);
cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
} else { // set the user specified locale failed, use default LC_CTYPE as current locale
locale = setlocale(LC_CTYPE, tsLocale);
tscInfo("failed to set locale:%s, current locale:%s", str, tsLocale);
}
tstrncpy(tsLocale, locale, TD_LOCALE_LEN);
char *charset = strrchr(tsLocale, sep);
if (charset != NULL) {
charset += 1;
charset = taosCharsetReplace(charset);
if (taosValidateEncodec(charset)) {
if (strlen(tsCharset) == 0) {
tscInfo("charset set:%s", charset);
} else {
tscInfo("charset changed from %s to %s", tsCharset, charset);
}
tstrncpy(tsCharset, charset, TD_LOCALE_LEN);
cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
} else {
tscInfo("charset:%s is not valid in locale, charset remains:%s", charset, tsCharset);
}
taosMemoryFree(charset);
} else { // it may be windows system
tscInfo("charset remains:%s", tsCharset);
}
} else {
tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str,
tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr);
}
case TSDB_OPTION_LOCALE:
pItem = cfgGetItem(pCfg, "locale");
break;
}
case TSDB_OPTION_CHARSET: {
/* set charset will override the value of charset, assigned during system locale changed */
cfg = taosGetConfigOption("charset");
assert(cfg != NULL);
size_t len = strlen(str);
if (len == 0 || len > TD_LOCALE_LEN) {
tscInfo("failed to set charset:%s", str);
return -1;
}
if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) {
if (taosValidateEncodec(str)) {
if (strlen(tsCharset) == 0) {
tscInfo("charset is set:%s", str);
} else {
tscInfo("charset changed from %s to %s", tsCharset, str);
}
tstrncpy(tsCharset, str, TD_LOCALE_LEN);
cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
} else {
tscInfo("charset:%s not valid", str);
}
} else {
tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str,
tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr);
}
case TSDB_OPTION_CHARSET:
pItem = cfgGetItem(pCfg, "charset");
break;
}
case TSDB_OPTION_TIMEZONE:
cfg = taosGetConfigOption("timezone");
assert(cfg != NULL);
if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) {
tstrncpy(tsTimezoneStr, str, TD_TIMEZONE_LEN);
tsSetTimeZone();
cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
tscDebug("timezone set:%s, input:%s by taos_options", tsTimezoneStr, str);
} else {
tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str,
tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr);
}
pItem = cfgGetItem(pCfg, "timezone");
break;
default:
// TODO return the correct error code to client in the format for taos_errstr()
tscError("Invalid option %d", option);
return -1;
break;
}
#endif
return 0;
if (pItem == NULL) {
tscError("Invalid option %d", option);
return -1;
}
int code = cfgSetItem(pCfg, pItem->name, str, CFG_STYPE_TAOS_OPTIONS);
if (code != 0) {
tscError("failed to set cfg:%s to %s since %s", pItem->name, str, terrstr());
} else {
tscInfo("set cfg:%s to %s", pItem->name, str);
}
return code;
}
/**

38
source/client/src/clientHb.c
View File

@ -630,10 +630,29 @@ void appHbMgrCleanup(void) {
int sz = taosArrayGetSize(clientHbMgr.appHbMgrs);
for (int i = 0; i < sz; i++) {
SAppHbMgr *pTarget = taosArrayGetP(clientHbMgr.appHbMgrs, i);
void *pIter = taosHashIterate(pTarget->activeInfo, NULL);
while (pIter != NULL) {
SClientHbReq *pOneReq = pIter;
hbFreeReq(pOneReq);
taosHashCleanup(pOneReq->info);
pIter = taosHashIterate(pTarget->activeInfo, pIter);
}
taosHashCleanup(pTarget->activeInfo);
pTarget->activeInfo = NULL;
pIter = taosHashIterate(pTarget->connInfo, NULL);
while (pIter != NULL) {
SHbConnInfo *info = pIter;
taosMemoryFree(info->param);
pIter = taosHashIterate(pTarget->connInfo, pIter);
}
taosHashCleanup(pTarget->connInfo);
pTarget->connInfo = NULL;
taosMemoryFree(pTarget->key);
taosMemoryFree(pTarget);
}
}
@ -716,12 +735,23 @@ int hbRegisterConn(SAppHbMgr *pAppHbMgr, int64_t tscRefId, int64_t clusterId, in
}
void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) {
int32_t code = 0;
code = taosHashRemove(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey));
code = taosHashRemove(pAppHbMgr->connInfo, &connKey, sizeof(SClientHbKey));
if (code) {
SClientHbReq *pReq = taosHashGet(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey));
if (pReq) {
hbFreeReq(pReq);
taosHashCleanup(pReq->info);
taosHashRemove(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey));
}
SHbConnInfo *info = taosHashGet(pAppHbMgr->connInfo, &connKey, sizeof(SClientHbKey));
if (info) {
taosMemoryFree(info->param);
taosHashRemove(pAppHbMgr->connInfo, &connKey, sizeof(SClientHbKey));
}
if (NULL == pReq || NULL == info) {
return;
}
atomic_sub_fetch_32(&pAppHbMgr->connKeyCnt, 1);
}

65
source/client/src/clientImpl.c
View File

@ -885,3 +885,68 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR
return setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows,
convertUcs4);
}
TSDB_SERVER_STATUS taos_check_server_status(const char* fqdn, int port, char* details, int maxlen) {
TSDB_SERVER_STATUS code = TSDB_SRV_STATUS_UNAVAILABLE;
void* clientRpc = NULL;
SServerStatusRsp statusRsp = {0};
SEpSet epSet = {.inUse = 0, .numOfEps = 1};
SRpcMsg rpcMsg = {.ahandle = (void*)0x9526, .msgType = TDMT_DND_SERVER_STATUS};
SRpcMsg rpcRsp = {0};
SRpcInit rpcInit = {0};
char pass[TSDB_PASSWORD_LEN + 1] = {0};
taosEncryptPass_c((uint8_t*)("_pwd"), strlen("_pwd"), pass);
rpcInit.label = "CHK";
rpcInit.numOfThreads = 1;
rpcInit.cfp = NULL;
rpcInit.sessions = 16;
rpcInit.connType = TAOS_CONN_CLIENT;
rpcInit.idleTime = tsShellActivityTimer * 1000;
rpcInit.user = "_dnd";
rpcInit.ckey = "_key";
rpcInit.spi = 1;
rpcInit.secret = pass;
clientRpc = rpcOpen(&rpcInit);
if (clientRpc == NULL) {
tscError("failed to init server status client");
goto _OVER;
}
if (fqdn == NULL) {
fqdn = tsLocalFqdn;
}
if (port == 0) {
port = tsServerPort;
}
tstrncpy(epSet.eps[0].fqdn, fqdn, TSDB_FQDN_LEN);
epSet.eps[0].port = (uint16_t)port;
rpcSendRecv(clientRpc, &epSet, &rpcMsg, &rpcRsp);
if (rpcRsp.code != 0 || rpcRsp.contLen <= 0 || rpcRsp.pCont == NULL) {
tscError("failed to send server status req since %s", terrstr());
goto _OVER;
}
if (tDeserializeSServerStatusRsp(rpcRsp.pCont, rpcRsp.contLen, &statusRsp) != 0) {
tscError("failed to parse server status rsp since %s", terrstr());
goto _OVER;
}
code = statusRsp.statusCode;
if (details != NULL && statusRsp.details != NULL) {
tstrncpy(details, statusRsp.details, maxlen);
}
_OVER:
if (clientRpc != NULL) {
rpcClose(clientRpc);
}
if (rpcRsp.pCont != NULL) {
rpcFreeCont(rpcRsp.pCont);
}
return code;
}

1
source/client/src/clientMain.c
View File

@ -71,6 +71,7 @@ void taos_cleanup(void) {
schedulerDestroy();
tscInfo("all local resources released");
taosCleanupCfg();
taosCloseLog();
}

8
source/client/src/clientMsgHandler.c
View File

@ -209,9 +209,9 @@ int32_t processDropDbRsp(void* param, const SDataBuf* pMsg, int32_t code) {
}
void initMsgHandleFp() {
handleRequestRspFp[TMSG_INDEX(TDMT_MND_CONNECT)] = processConnectRsp;
handleRequestRspFp[TMSG_INDEX(TDMT_MND_CREATE_DB)] = processCreateDbRsp;
handleRequestRspFp[TMSG_INDEX(TDMT_MND_USE_DB)] = processUseDbRsp;
handleRequestRspFp[TMSG_INDEX(TDMT_MND_CONNECT)] = processConnectRsp;
handleRequestRspFp[TMSG_INDEX(TDMT_MND_CREATE_DB)] = processCreateDbRsp;
handleRequestRspFp[TMSG_INDEX(TDMT_MND_USE_DB)] = processUseDbRsp;
handleRequestRspFp[TMSG_INDEX(TDMT_MND_CREATE_STB)] = processCreateTableRsp;
handleRequestRspFp[TMSG_INDEX(TDMT_MND_DROP_DB)] = processDropDbRsp;
handleRequestRspFp[TMSG_INDEX(TDMT_MND_DROP_DB)] = processDropDbRsp;
}

18
source/common/src/tdatablock.c
View File

@ -216,6 +216,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co
if (pSource->hasNull) {
pColumnInfoData->hasNull = pSource->hasNull;
}
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
// Handle the bitmap
char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * (numOfRow1 + numOfRow2));
@ -1088,8 +1089,8 @@ void blockDataCleanup(SSDataBlock* pDataBlock) {
}
}
int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows) {
if (0 == numOfRows) {
int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, size_t existRows, uint32_t numOfRows) {
if (0 == numOfRows || numOfRows <= existRows) {
return TSDB_CODE_SUCCESS;
}
@ -1100,19 +1101,16 @@ int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows)
}
pColumn->varmeta.offset = (int32_t*)tmp;
memset(pColumn->varmeta.offset, 0, sizeof(int32_t) * numOfRows);
pColumn->varmeta.length = 0;
pColumn->varmeta.allocLen = 0;
taosMemoryFreeClear(pColumn->pData);
memset(&pColumn->varmeta.offset[existRows], 0, sizeof(int32_t) * (numOfRows - existRows));
} else {
char* tmp = taosMemoryRealloc(pColumn->nullbitmap, BitmapLen(numOfRows));
if (tmp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t oldLen = BitmapLen(existRows);
pColumn->nullbitmap = tmp;
memset(pColumn->nullbitmap, 0, BitmapLen(numOfRows));
memset(&pColumn->nullbitmap[oldLen], 0, BitmapLen(numOfRows) - oldLen);
if (pColumn->info.type == TSDB_DATA_TYPE_NULL) {
return TSDB_CODE_SUCCESS;
@ -1148,7 +1146,7 @@ int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) {
for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) {
SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
code = colInfoDataEnsureCapacity(p, numOfRows);
code = colInfoDataEnsureCapacity(p, pDataBlock->info.rows, numOfRows);
if (code) {
return code;
}
@ -1193,7 +1191,7 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) {
SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i);
SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i);
int32_t code = colInfoDataEnsureCapacity(pDst, pDataBlock->info.rows);
int32_t code = colInfoDataEnsureCapacity(pDst, 0, pDataBlock->info.rows);
if (code != TSDB_CODE_SUCCESS) {
return NULL;
}

206
source/common/src/tmsg.c
View File

@ -52,6 +52,10 @@ int32_t tGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) {
if (pIter->len == 0) {
pIter->len += sizeof(SSubmitReq);
} else {
if (pIter->len >= pIter->totalLen) {
ASSERT(0);
}
SSubmitBlk *pSubmitBlk = (SSubmitBlk *)POINTER_SHIFT(pIter->pMsg, pIter->len);
pIter->len += (sizeof(SSubmitBlk) + pSubmitBlk->dataLen + pSubmitBlk->schemaLen);
ASSERT(pIter->len > 0);
@ -397,8 +401,6 @@ int32_t tDeserializeSClientHbBatchRsp(void *buf, int32_t bufLen, SClientHbBatchR
int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
int32_t tlen = 0;
tlen += taosEncodeFixedI64(buf, pReq->ver);
tlen += taosEncodeString(buf, pReq->dbFName);
tlen += taosEncodeString(buf, pReq->name);
tlen += taosEncodeFixedU32(buf, pReq->ttl);
tlen += taosEncodeFixedU32(buf, pReq->keep);
@ -411,7 +413,7 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nBSmaCols);
for (col_id_t i = 0; i < pReq->stbCfg.nCols; ++i) {
tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pSchema[i].type);
tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pSchema[i].sma);
tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pSchema[i].flags);
tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pSchema[i].colId);
tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pSchema[i].bytes);
tlen += taosEncodeString(buf, pReq->stbCfg.pSchema[i].name);
@ -419,7 +421,7 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nTagCols);
for (col_id_t i = 0; i < pReq->stbCfg.nTagCols; ++i) {
tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pTagSchema[i].type);
tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pTagSchema[i].index);
tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pTagSchema[i].flags);
tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pTagSchema[i].colId);
tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pTagSchema[i].bytes);
tlen += taosEncodeString(buf, pReq->stbCfg.pTagSchema[i].name);
@ -443,7 +445,7 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.nBSmaCols);
for (col_id_t i = 0; i < pReq->ntbCfg.nCols; ++i) {
tlen += taosEncodeFixedI8(buf, pReq->ntbCfg.pSchema[i].type);
tlen += taosEncodeFixedI8(buf, pReq->ntbCfg.pSchema[i].sma);
tlen += taosEncodeFixedI8(buf, pReq->ntbCfg.pSchema[i].flags);
tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.pSchema[i].colId);
tlen += taosEncodeFixedI32(buf, pReq->ntbCfg.pSchema[i].bytes);
tlen += taosEncodeString(buf, pReq->ntbCfg.pSchema[i].name);
@ -466,8 +468,6 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
}
void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
buf = taosDecodeFixedI64(buf, &(pReq->ver));
buf = taosDecodeString(buf, &(pReq->dbFName));
buf = taosDecodeString(buf, &(pReq->name));
buf = taosDecodeFixedU32(buf, &(pReq->ttl));
buf = taosDecodeFixedU32(buf, &(pReq->keep));
@ -478,10 +478,10 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
buf = taosDecodeFixedI64(buf, &(pReq->stbCfg.suid));
buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nCols));
buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nBSmaCols));
pReq->stbCfg.pSchema = (SSchemaEx *)taosMemoryMalloc(pReq->stbCfg.nCols * sizeof(SSchemaEx));
pReq->stbCfg.pSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nCols * sizeof(SSchema));
for (col_id_t i = 0; i < pReq->stbCfg.nCols; ++i) {
buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].type));
buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].sma));
buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].flags));
buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.pSchema[i].colId));
buf = taosDecodeFixedI32(buf, &(pReq->stbCfg.pSchema[i].bytes));
buf = taosDecodeStringTo(buf, pReq->stbCfg.pSchema[i].name);
@ -490,7 +490,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
pReq->stbCfg.pTagSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nTagCols * sizeof(SSchema));
for (col_id_t i = 0; i < pReq->stbCfg.nTagCols; ++i) {
buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pTagSchema[i].type));
buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pTagSchema[i].index));
buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pTagSchema[i].flags));
buf = taosDecodeFixedI16(buf, &pReq->stbCfg.pTagSchema[i].colId);
buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].bytes);
buf = taosDecodeStringTo(buf, pReq->stbCfg.pTagSchema[i].name);
@ -520,10 +520,10 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
case TD_NORMAL_TABLE:
buf = taosDecodeFixedI16(buf, &pReq->ntbCfg.nCols);
buf = taosDecodeFixedI16(buf, &(pReq->ntbCfg.nBSmaCols));
pReq->ntbCfg.pSchema = (SSchemaEx *)taosMemoryMalloc(pReq->ntbCfg.nCols * sizeof(SSchemaEx));
pReq->ntbCfg.pSchema = (SSchema *)taosMemoryMalloc(pReq->ntbCfg.nCols * sizeof(SSchema));
for (col_id_t i = 0; i < pReq->ntbCfg.nCols; ++i) {
buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].type);
buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].sma);
buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].flags);
buf = taosDecodeFixedI16(buf, &pReq->ntbCfg.pSchema[i].colId);
buf = taosDecodeFixedI32(buf, &pReq->ntbCfg.pSchema[i].bytes);
buf = taosDecodeStringTo(buf, pReq->ntbCfg.pSchema[i].name);
@ -611,6 +611,8 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq
if (tEncodeI32(&encoder, pReq->numOfTags) < 0) return -1;
if (tEncodeI32(&encoder, pReq->numOfSmas) < 0) return -1;
if (tEncodeI32(&encoder, pReq->commentLen) < 0) return -1;
if (tEncodeI32(&encoder, pReq->ast1Len) < 0) return -1;
if (tEncodeI32(&encoder, pReq->ast2Len) < 0) return -1;
for (int32_t i = 0; i < pReq->numOfColumns; ++i) {
SField *pField = taosArrayGet(pReq->pColumns, i);
@ -636,6 +638,12 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq
if (pReq->commentLen > 0) {
if (tEncodeBinary(&encoder, pReq->comment, pReq->commentLen) < 0) return -1;
}
if (pReq->ast1Len > 0) {
if (tEncodeBinary(&encoder, pReq->pAst1, pReq->ast1Len) < 0) return -1;
}
if (pReq->ast2Len > 0) {
if (tEncodeBinary(&encoder, pReq->pAst2, pReq->ast2Len) < 0) return -1;
}
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
@ -658,6 +666,8 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
if (tDecodeI32(&decoder, &pReq->numOfTags) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->numOfSmas) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->commentLen) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->ast1Len) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->ast2Len) < 0) return -1;
pReq->pColumns = taosArrayInit(pReq->numOfColumns, sizeof(SField));
pReq->pTags = taosArrayInit(pReq->numOfTags, sizeof(SField));
@ -706,6 +716,18 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
if (tDecodeCStrTo(&decoder, pReq->comment) < 0) return -1;
}
if (pReq->ast1Len > 0) {
pReq->pAst1 = taosMemoryMalloc(pReq->ast1Len);
if (pReq->pAst1 == NULL) return -1;
if (tDecodeCStrTo(&decoder, pReq->pAst1) < 0) return -1;
}
if (pReq->ast2Len > 0) {
pReq->pAst2 = taosMemoryMalloc(pReq->ast2Len);
if (pReq->pAst2 == NULL) return -1;
if (tDecodeCStrTo(&decoder, pReq->pAst2) < 0) return -1;
}
tEndDecode(&decoder);
tCoderClear(&decoder);
@ -717,6 +739,8 @@ void tFreeSMCreateStbReq(SMCreateStbReq *pReq) {
taosArrayDestroy(pReq->pTags);
taosArrayDestroy(pReq->pSmas);
taosMemoryFreeClear(pReq->comment);
taosMemoryFreeClear(pReq->pAst1);
taosMemoryFreeClear(pReq->pAst2);
pReq->pColumns = NULL;
pReq->pTags = NULL;
pReq->pSmas = NULL;
@ -1003,7 +1027,7 @@ int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
for (int32_t i = 0; i < vlen; ++i) {
SVnodeLoad *pload = taosArrayGet(pReq->pVloads, i);
if (tEncodeI32(&encoder, pload->vgId) < 0) return -1;
if (tEncodeI8(&encoder, pload->role) < 0) return -1;
if (tEncodeI32(&encoder, pload->syncState) < 0) return -1;
if (tEncodeI64(&encoder, pload->numOfTables) < 0) return -1;
if (tEncodeI64(&encoder, pload->numOfTimeSeries) < 0) return -1;
if (tEncodeI64(&encoder, pload->totalStorage) < 0) return -1;
@ -1054,7 +1078,7 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
for (int32_t i = 0; i < vlen; ++i) {
SVnodeLoad vload = {0};
if (tDecodeI32(&decoder, &vload.vgId) < 0) return -1;
if (tDecodeI8(&decoder, &vload.role) < 0) return -1;
if (tDecodeI32(&decoder, &vload.syncState) < 0) return -1;
if (tDecodeI64(&decoder, &vload.numOfTables) < 0) return -1;
if (tDecodeI64(&decoder, &vload.numOfTimeSeries) < 0) return -1;
if (tDecodeI64(&decoder, &vload.totalStorage) < 0) return -1;
@ -1736,7 +1760,7 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) {
if (tEncodeI8(&encoder, pReq->precision) < 0) return -1;
if (tEncodeI8(&encoder, pReq->compression) < 0) return -1;
if (tEncodeI8(&encoder, pReq->replications) < 0) return -1;
if (tEncodeI8(&encoder, pReq->quorum) < 0) return -1;
if (tEncodeI8(&encoder, pReq->strict) < 0) return -1;
if (tEncodeI8(&encoder, pReq->update) < 0) return -1;
if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1;
if (tEncodeI8(&encoder, pReq->ignoreExist) < 0) return -1;
@ -1779,7 +1803,7 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq)
if (tDecodeI8(&decoder, &pReq->precision) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->compression) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->replications) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->quorum) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->update) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->ignoreExist) < 0) return -1;
@ -1827,7 +1851,7 @@ int32_t tSerializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) {
if (tEncodeI32(&encoder, pReq->daysToKeep2) < 0) return -1;
if (tEncodeI32(&encoder, pReq->fsyncPeriod) < 0) return -1;
if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1;
if (tEncodeI8(&encoder, pReq->quorum) < 0) return -1;
if (tEncodeI8(&encoder, pReq->strict) < 0) return -1;
if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1;
if (tEncodeI8(&encoder, pReq->replications) < 0) return -1;
tEndEncode(&encoder);
@ -1849,7 +1873,7 @@ int32_t tDeserializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) {
if (tDecodeI32(&decoder, &pReq->daysToKeep2) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->fsyncPeriod) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->quorum) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->replications) < 0) return -1;
tEndDecode(&decoder);
@ -2006,7 +2030,7 @@ int32_t tDeserializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp
void tFreeSQnodeListRsp(SQnodeListRsp *pRsp) { taosArrayDestroy(pRsp->epSetList); }
int32_t tSerializeSSyncDbReq(void *buf, int32_t bufLen, SSyncDbReq *pReq) {
int32_t tSerializeSCompactDbReq(void *buf, int32_t bufLen, SCompactDbReq *pReq) {
SCoder encoder = {0};
tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
@ -2019,7 +2043,7 @@ int32_t tSerializeSSyncDbReq(void *buf, int32_t bufLen, SSyncDbReq *pReq) {
return tlen;
}
int32_t tDeserializeSSyncDbReq(void *buf, int32_t bufLen, SSyncDbReq *pReq) {
int32_t tDeserializeSCompactDbReq(void *buf, int32_t bufLen, SCompactDbReq *pReq) {
SCoder decoder = {0};
tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
@ -2207,10 +2231,18 @@ int32_t tSerializeSDbCfgRsp(void *buf, int32_t bufLen, const SDbCfgRsp *pRsp) {
if (tEncodeI8(&encoder, pRsp->precision) < 0) return -1;
if (tEncodeI8(&encoder, pRsp->compression) < 0) return -1;
if (tEncodeI8(&encoder, pRsp->replications) < 0) return -1;
if (tEncodeI8(&encoder, pRsp->quorum) < 0) return -1;
if (tEncodeI8(&encoder, pRsp->strict) < 0) return -1;
if (tEncodeI8(&encoder, pRsp->update) < 0) return -1;
if (tEncodeI8(&encoder, pRsp->cacheLastRow) < 0) return -1;
if (tEncodeI8(&encoder, pRsp->streamMode) < 0) return -1;
if (tEncodeI32(&encoder, pRsp->numOfRetensions) < 0) return -1;
for (int32_t i = 0; i < pRsp->numOfRetensions; ++i) {
SRetention *pRetension = taosArrayGet(pRsp->pRetensions, i);
if (tEncodeI32(&encoder, pRetension->freq) < 0) return -1;
if (tEncodeI32(&encoder, pRetension->keep) < 0) return -1;
if (tEncodeI8(&encoder, pRetension->freqUnit) < 0) return -1;
if (tEncodeI8(&encoder, pRetension->keepUnit) < 0) return -1;
}
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
@ -2238,11 +2270,28 @@ int32_t tDeserializeSDbCfgRsp(void *buf, int32_t bufLen, SDbCfgRsp *pRsp) {
if (tDecodeI8(&decoder, &pRsp->precision) < 0) return -1;
if (tDecodeI8(&decoder, &pRsp->compression) < 0) return -1;
if (tDecodeI8(&decoder, &pRsp->replications) < 0) return -1;
if (tDecodeI8(&decoder, &pRsp->quorum) < 0) return -1;
if (tDecodeI8(&decoder, &pRsp->strict) < 0) return -1;
if (tDecodeI8(&decoder, &pRsp->update) < 0) return -1;
if (tDecodeI8(&decoder, &pRsp->cacheLastRow) < 0) return -1;
if (tDecodeI8(&decoder, &pRsp->streamMode) < 0) return -1;
if (tDecodeI32(&decoder, &pRsp->numOfRetensions) < 0) return -1;
pRsp->pRetensions = taosArrayInit(pRsp->numOfRetensions, sizeof(SRetention));
if (pRsp->pRetensions == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
for (int32_t i = 0; i < pRsp->numOfRetensions; ++i) {
SRetention rentension = {0};
if (tDecodeI32(&decoder, &rentension.freq) < 0) return -1;
if (tDecodeI32(&decoder, &rentension.keep) < 0) return -1;
if (tDecodeI8(&decoder, &rentension.freqUnit) < 0) return -1;
if (tDecodeI8(&decoder, &rentension.keepUnit) < 0) return -1;
if (taosArrayPush(pRsp->pRetensions, &rentension) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
}
tEndDecode(&decoder);
tCoderClear(&decoder);
@ -2831,7 +2880,7 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR
if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1;
if (tEncodeI8(&encoder, pReq->precision) < 0) return -1;
if (tEncodeI8(&encoder, pReq->compression) < 0) return -1;
if (tEncodeI8(&encoder, pReq->quorum) < 0) return -1;
if (tEncodeI8(&encoder, pReq->strict) < 0) return -1;
if (tEncodeI8(&encoder, pReq->update) < 0) return -1;
if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1;
if (tEncodeI8(&encoder, pReq->replica) < 0) return -1;
@ -2882,7 +2931,7 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *
if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->precision) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->compression) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->quorum) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->update) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->replica) < 0) return -1;
@ -2954,6 +3003,87 @@ int32_t tDeserializeSDropVnodeReq(void *buf, int32_t bufLen, SDropVnodeReq *pReq
return 0;
}
int32_t tSerializeSCompactVnodeReq(void *buf, int32_t bufLen, SCompactVnodeReq *pReq) {
SCoder encoder = {0};
tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI64(&encoder, pReq->dbUid) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tCoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSCompactVnodeReq(void *buf, int32_t bufLen, SCompactVnodeReq *pReq) {
SCoder decoder = {0};
tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI64(&decoder, &pReq->dbUid) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
tEndDecode(&decoder);
tCoderClear(&decoder);
return 0;
}
int32_t tSerializeSAlterVnodeReq(void *buf, int32_t bufLen, SAlterVnodeReq *pReq) {
SCoder encoder = {0};
tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pReq->vgVersion) < 0) return -1;
if (tEncodeI32(&encoder, pReq->totalBlocks) < 0) return -1;
if (tEncodeI32(&encoder, pReq->daysToKeep0) < 0) return -1;
if (tEncodeI32(&encoder, pReq->daysToKeep1) < 0) return -1;
if (tEncodeI32(&encoder, pReq->daysToKeep2) < 0) return -1;
if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1;
if (tEncodeI8(&encoder, pReq->strict) < 0) return -1;
if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1;
if (tEncodeI8(&encoder, pReq->replica) < 0) return -1;
if (tEncodeI8(&encoder, pReq->selfIndex) < 0) return -1;
for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) {
SReplica *pReplica = &pReq->replicas[i];
if (tEncodeSReplica(&encoder, pReplica) < 0) return -1;
}
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tCoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSAlterVnodeReq(void *buf, int32_t bufLen, SAlterVnodeReq *pReq) {
SCoder decoder = {0};
tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->vgVersion) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->totalBlocks) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->daysToKeep0) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->daysToKeep1) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->daysToKeep2) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->replica) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->selfIndex) < 0) return -1;
for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) {
SReplica *pReplica = &pReq->replicas[i];
if (tDecodeSReplica(&decoder, pReplica) < 0) return -1;
}
tEndDecode(&decoder);
tCoderClear(&decoder);
return 0;
}
int32_t tSerializeSKillQueryReq(void *buf, int32_t bufLen, SKillQueryReq *pReq) {
SCoder encoder = {0};
tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
@ -3099,6 +3229,34 @@ int32_t tDeserializeSAuthReq(void *buf, int32_t bufLen, SAuthReq *pReq) {
return 0;
}
int32_t tSerializeSServerStatusRsp(void *buf, int32_t bufLen, SServerStatusRsp *pRsp) {
SCoder encoder = {0};
tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pRsp->statusCode) < 0) return -1;
if (tEncodeCStr(&encoder, pRsp->details) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tCoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSServerStatusRsp(void *buf, int32_t bufLen, SServerStatusRsp *pRsp) {
SCoder decoder = {0};
tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pRsp->statusCode) < 0) return -1;
if (tDecodeCStrTo(&decoder, pRsp->details) < 0) return -1;
tEndDecode(&decoder);
tCoderClear(&decoder);
return 0;
}
int32_t tEncodeSMqOffset(SCoder *encoder, const SMqOffset *pOffset) {
if (tEncodeI32(encoder, pOffset->vgId) < 0) return -1;
if (tEncodeI64(encoder, pOffset->offset) < 0) return -1;

4
source/common/src/tmsgcb.c
View File

@ -41,3 +41,7 @@ void tmsgRegisterBrokenLinkArg(const SMsgCb* pMsgCb, SRpcMsg* pMsg) {
void tmsgReleaseHandle(void* handle, int8_t type) {
(*tsDefaultMsgCb.releaseHandleFp)(tsDefaultMsgCb.pWrapper, handle, type);
}
void tmsgReportStartup(const char* name, const char* desc) {
(*tsDefaultMsgCb.reportStartupFp)(tsDefaultMsgCb.pWrapper, name, desc);
}

2
source/dnode/mgmt/implement/inc/dmImp.h
View File

@ -49,6 +49,7 @@ int32_t dmProcessDropNodeReq(SDnode *pDnode, EDndNodeType ntype, SNodeMsg *pMsg)
// dmMonitor.c
void dmGetVnodeLoads(SDnode *pDnode, SMonVloadInfo *pInfo);
void dmGetMnodeLoads(SDnode *pDnode, SMonMloadInfo *pInfo);
void dmSendMonitorReport(SDnode *pDnode);
// dmWorker.c
@ -70,6 +71,7 @@ void vmSetMgmtFp(SMgmtWrapper *pWrapper);
void mmSetMgmtFp(SMgmtWrapper *pWrapper);
void vmGetVnodeLoads(SMgmtWrapper *pWrapper, SMonVloadInfo *pInfo);
void mmGetMnodeLoads(SMgmtWrapper *pWrapper, SMonMloadInfo *pInfo);
void mmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonMmInfo *mmInfo);
void vmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonVmInfo *vmInfo);
void qmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonQmInfo *qmInfo);

17
source/dnode/mgmt/implement/src/dmExec.c
View File

@ -140,6 +140,7 @@ int32_t dmOpenNode(SMgmtWrapper *pWrapper) {
if (dmRunParentProc(pWrapper) != 0) return -1;
}
dmReportStartup(pWrapper->pDnode, pWrapper->name, "openned");
return 0;
}
@ -161,6 +162,7 @@ int32_t dmStartNode(SMgmtWrapper *pWrapper) {
}
}
dmReportStartup(pWrapper->pDnode, pWrapper->name, "started");
return 0;
}
@ -172,6 +174,11 @@ void dmStopNode(SMgmtWrapper *pWrapper) {
void dmCloseNode(SMgmtWrapper *pWrapper) {
dInfo("node:%s, start to close", pWrapper->name);
while (pWrapper->refCount > 0) {
taosMsleep(10);
}
if (pWrapper->procType == DND_PROC_PARENT) {
if (pWrapper->procId > 0 && taosProcExist(pWrapper->procId)) {
dInfo("node:%s, send kill signal to the child process:%d", pWrapper->name, pWrapper->procId);
@ -184,18 +191,10 @@ void dmCloseNode(SMgmtWrapper *pWrapper) {
dmStopNode(pWrapper);
pWrapper->required = false;
taosWLockLatch(&pWrapper->latch);
if (pWrapper->deployed) {
(*pWrapper->fp.closeFp)(pWrapper);
pWrapper->deployed = false;
}
(*pWrapper->fp.closeFp)(pWrapper);
taosWUnLockLatch(&pWrapper->latch);
while (pWrapper->refCount > 0) {
taosMsleep(10);
}
if (pWrapper->procObj) {
taosProcCleanup(pWrapper->procObj);
pWrapper->procObj = NULL;

24
source/dnode/mgmt/implement/src/dmHandle.c
View File

@ -42,6 +42,7 @@ static int32_t dmProcessStatusRsp(SDnode *pDnode, SRpcMsg *pRsp) {
dmUpdateDnodeCfg(pDnode, &statusRsp.dnodeCfg);
dmUpdateEps(pDnode, statusRsp.pDnodeEps);
}
rpcFreeCont(pRsp->pCont);
tFreeSStatusRsp(&statusRsp);
}
@ -71,9 +72,23 @@ void dmSendStatusReq(SDnode *pDnode) {
memcpy(req.clusterCfg.charset, tsCharset, TD_LOCALE_LEN);
taosRUnLockLatch(&pDnode->data.latch);
SMonVloadInfo info = {0};
dmGetVnodeLoads(pDnode, &info);
req.pVloads = info.pVloads;
SMonVloadInfo vinfo = {0};
dmGetVnodeLoads(pDnode, &vinfo);
req.pVloads = vinfo.pVloads;
pDnode->data.unsyncedVgId = 0;
pDnode->data.vndState = TAOS_SYNC_STATE_LEADER;
for (int32_t i = 0; i < taosArrayGetSize(req.pVloads); ++i) {
SVnodeLoad *pLoad = taosArrayGet(req.pVloads, i);
if (pLoad->syncState != TAOS_SYNC_STATE_LEADER && pLoad->syncState != TAOS_SYNC_STATE_FOLLOWER) {
pDnode->data.unsyncedVgId = pLoad->vgId;
pDnode->data.vndState = pLoad->syncState;
}
}
SMonMloadInfo minfo = {0};
dmGetMnodeLoads(pDnode, &minfo);
pDnode->data.isMnode = minfo.isMnode;
pDnode->data.mndState = minfo.load.syncState;
int32_t contLen = tSerializeSStatusReq(NULL, 0, &req);
void *pHead = rpcMallocCont(contLen);
@ -160,9 +175,9 @@ int32_t dmProcessDropNodeReq(SDnode *pDnode, EDndNodeType ntype, SNodeMsg *pMsg)
dmReleaseWrapper(pWrapper);
if (code == 0) {
dmCloseNode(pWrapper);
pWrapper->required = false;
pWrapper->deployed = false;
dmCloseNode(pWrapper);
taosRemoveDir(pWrapper->path);
}
taosThreadMutexUnlock(&pDnode->mutex);
@ -230,6 +245,7 @@ static int32_t dmInitMgmt(SMgmtWrapper *pWrapper) {
dError("failed to init transport since %s", terrstr());
return -1;
}
dmReportStartup(pDnode, "dnode-transport", "initialized");
dInfo("dnode-mgmt is initialized");
return 0;

26
source/dnode/mgmt/implement/src/dmMonitor.c
View File

@ -183,3 +183,29 @@ void dmGetVnodeLoads(SDnode *pDnode, SMonVloadInfo *pInfo) {
}
dmReleaseWrapper(pWrapper);
}
void dmGetMnodeLoads(SDnode *pDnode, SMonMloadInfo *pInfo) {
SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, MNODE);
if (pWrapper == NULL) {
pInfo->isMnode = 0;
return;
}
bool getFromAPI = !tsMultiProcess;
if (getFromAPI) {
mmGetMnodeLoads(pWrapper, pInfo);
} else {
SRpcMsg req = {.msgType = TDMT_MON_MM_LOAD};
SRpcMsg rsp = {0};
SEpSet epset = {.inUse = 0, .numOfEps = 1};
tstrncpy(epset.eps[0].fqdn, pDnode->data.localFqdn, TSDB_FQDN_LEN);
epset.eps[0].port = tsServerPort;
dmSendRecv(pDnode, &epset, &req, &rsp);
if (rsp.code == 0 && rsp.contLen > 0) {
tDeserializeSMonMloadInfo(rsp.pCont, rsp.contLen, pInfo);
}
rpcFreeCont(rsp.pCont);
}
dmReleaseWrapper(pWrapper);
}

Some files were not shown because too many files have changed in this diff Show More