xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into test/jcy

This commit is contained in:
jiacy-jcy 2022-04-25 10:46:13 +08:00
parent a5daa3f4b8 8ad6d4606c
commit fa36a1725d
104 changed files with 5235 additions and 5559 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Jenkinsfile vendored
View File

@ -113,7 +113,7 @@ pipeline {
'''
sh'''
cd ${WKC}/debug
ctest
ctest -VV
'''
}
}

20
Jenkinsfile2
View File

@ -88,6 +88,12 @@ def pre_test(){
cmake .. > /dev/null
make -j4> /dev/null
'''
sh'''
cd ${WKPY}
git reset --hard
git pull
pip3 install .
'''
return 1
}
@ -97,6 +103,7 @@ pipeline {
environment{
WK = '/var/lib/jenkins/workspace/TDinternal'
WKC= '/var/lib/jenkins/workspace/TDengine'
WKPY= '/var/lib/jenkins/workspace/taos-connector-python'
}
stages {
stage('pre_build'){
@ -113,12 +120,17 @@ pipeline {
timeout(time: 45, unit: 'MINUTES'){
pre_test()
sh'''
cd ${WKC}/tests
./test-all.sh b1fq
cd ${WKC}/debug
ctest -VV
'''
sh'''
cd ${WKC}/debug
ctest
export LD_LIBRARY_PATH=${WKC}/debug/build/lib
cd ${WKC}/tests/system-test
./fulltest.sh
'''
sh'''
cd ${WKC}/tests
./test-all.sh b1fq
'''
}
}

2
cmake/cmake.define
View File

@ -1,6 +1,6 @@
cmake_minimum_required(VERSION 3.16)
set(CMAKE_VERBOSE_MAKEFILE ON)
set(CMAKE_VERBOSE_MAKEFILE OFF)
#set output directory
SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib)

6
examples/c/asyncdemo.c
View File

@ -45,7 +45,7 @@ typedef struct {
void taos_insert_call_back(void *param, TAOS_RES *tres, int code);
void taos_select_call_back(void *param, TAOS_RES *tres, int code);
void taos_error(TAOS *taos);
void shellPrintError(TAOS *taos);
static void queryDB(TAOS *taos, char *command) {
int i;
@ -102,7 +102,7 @@ int main(int argc, char *argv[])
taos = taos_connect(argv[1], "root", "taosdata", NULL, 0);
if (taos == NULL)
taos_error(taos);
shellPrintError(taos);
printf("success to connect to server\n");
@ -193,7 +193,7 @@ int main(int argc, char *argv[])
return 0;
}
void taos_error(TAOS *con)
void shellPrintError(TAOS *con)
{
fprintf(stderr, "TDengine error: %s\n", taos_errstr(con));
taos_close(con);

4
include/common/tdatablock.h
View File

@ -74,8 +74,8 @@ static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData,
}
char *data = colDataGetVarData(pColumnInfoData, row);
return (*data == TSDB_DATA_TYPE_NULL);
}
}
if (!pColumnInfoData->hasNull) {
return false;
}

7
include/common/tglobal.h
View File

@ -34,13 +34,9 @@ extern int32_t tsVersion;
extern int32_t tsStatusInterval;
// common
extern int32_t tsRpcTimer;
extern int32_t tsRpcMaxTime;
extern bool tsRpcForceTcp; // all commands go to tcp protocol if this is enabled
extern int32_t tsMaxConnections;
extern int32_t tsMaxShellConns;
extern int32_t tsShellActivityTimer;
extern int32_t tsMaxTmrCtrl;
extern int32_t tsCompressMsgSize;
extern int32_t tsCompressColData;
extern int32_t tsMaxNumOfDistinctResults;
@ -98,9 +94,6 @@ extern bool tsDeadLockKillQuery;
extern int32_t tsQueryPolicy;
// client
extern int32_t tsMaxWildCardsLen;
extern int32_t tsMaxRegexStringLen;
extern int32_t tsMaxNumOfOrderedResults;
extern int32_t tsMinSlidingTime;
extern int32_t tsMinIntervalTime;
extern int32_t tsMaxStreamComputDelay;

1
include/common/tmsgdef.h
View File

@ -85,6 +85,7 @@ enum {
TD_DEF_MSG_TYPE(TDMT_DND_DROP_VNODE, "dnode-drop-vnode", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_DND_CONFIG_DNODE, "dnode-config-dnode", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_DND_SERVER_STATUS, "dnode-server-status", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_DND_NET_TEST, "dnode-net-test", NULL, NULL)
// Requests handled by MNODE
TD_NEW_MSG_SEG(TDMT_MND_MSG)

7
include/libs/nodes/plannodes.h
View File

@ -46,7 +46,7 @@ typedef struct SScanLogicNode {
struct STableMeta* pMeta;
SVgroupsInfo* pVgroupList;
EScanType scanType;
uint8_t scanFlag; // denotes reversed scan of data or not
uint8_t scanSeq[2]; // first is scan count, and second is reverse scan count
STimeWindow scanRange;
SName tableName;
bool showRewrite;
@ -189,9 +189,6 @@ typedef struct SScanPhysiNode {
SNodeList* pScanCols;
uint64_t uid; // unique id of the table
int8_t tableType;
int32_t order; // scan order: TSDB_ORDER_ASC|TSDB_ORDER_DESC
int32_t count; // repeat count
int32_t reverse; // reverse scan count
SName tableName;
} SScanPhysiNode;
@ -207,7 +204,7 @@ typedef struct SSystemTableScanPhysiNode {
typedef struct STableScanPhysiNode {
SScanPhysiNode scan;
uint8_t scanFlag; // denotes reversed scan of data or not
uint8_t scanSeq[2]; // first is scan count, and second is reverse scan count
STimeWindow scanRange;
double ratio;
int32_t dataRequired;

2
include/libs/planner/planner.h
View File

@ -37,6 +37,8 @@ typedef struct SPlanContext {
bool isStmtQuery;
void* pTransporter;
struct SCatalog* pCatalog;
char* pMsg;
int32_t msgLen;
} SPlanContext;
// Create the physical plan for the query, according to the AST.

23
include/libs/transport/trpc.h
View File

@ -38,11 +38,12 @@ typedef struct SRpcConnInfo {
typedef struct SRpcMsg {
tmsg_t msgType;
void *pCont;
void * pCont;
int contLen;
int32_t code;
void *handle; // rpc handle returned to app
void *ahandle; // app handle set by client
void * handle; // rpc handle returned to app
void * ahandle; // app handle set by client
int64_t refId; // refid, used by server
int noResp; // has response or not(default 0, 0: resp, 1: no resp);
int persistHandle; // persist handle or not
@ -54,8 +55,8 @@ typedef struct {
uint16_t clientPort;
SRpcMsg rpcMsg;
int32_t rspLen;
void *pRsp;
void *pNode;
void * pRsp;
void * pNode;
} SNodeMsg;
typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *);
@ -64,7 +65,7 @@ typedef int (*RpcRfp)(void *parent, SRpcMsg *, SEpSet *);
typedef struct SRpcInit {
uint16_t localPort; // local port
char *label; // for debug purpose
char * label; // for debug purpose
int numOfThreads; // number of threads to handle connections
int sessions; // number of sessions allowed
int8_t connType; // TAOS_CONN_UDP, TAOS_CONN_TCPC, TAOS_CONN_TCPS
@ -97,23 +98,23 @@ typedef struct {
typedef struct {
int32_t msgType;
void *val;
void * val;
int32_t (*clone)(void *src, void **dst);
void (*freeFunc)(const void *arg);
} SRpcBrokenlinkVal;
typedef struct {
SHashObj *args;
SHashObj * args;
SRpcBrokenlinkVal brokenVal;
} SRpcCtx;
int32_t rpcInit();
void rpcCleanup();
void *rpcOpen(const SRpcInit *pRpc);
void * rpcOpen(const SRpcInit *pRpc);
void rpcClose(void *);
void *rpcMallocCont(int contLen);
void * rpcMallocCont(int contLen);
void rpcFreeCont(void *pCont);
void *rpcReallocCont(void *ptr, int contLen);
void * rpcReallocCont(void *ptr, int contLen);
// Because taosd supports multi-process mode
// These functions should not be used on the server side

14
include/os/osSystem.h
View File

@ -31,19 +31,19 @@ extern "C" {
typedef struct TdCmd *TdCmdPtr;
TdCmdPtr taosOpenCmd(const char *cmd);
int64_t taosGetLineCmd(TdCmdPtr pCmd, char ** __restrict ptrBuf);
int32_t taosEOFCmd(TdCmdPtr pCmd);
int64_t taosCloseCmd(TdCmdPtr *ppCmd);
TdCmdPtr taosOpenCmd(const char* cmd);
int64_t taosGetLineCmd(TdCmdPtr pCmd, char** __restrict ptrBuf);
int32_t taosEOFCmd(TdCmdPtr pCmd);
int64_t taosCloseCmd(TdCmdPtr* ppCmd);
void* taosLoadDll(const char* filename);
void* taosLoadSym(void* handle, char* name);
void taosCloseDll(void* handle);
int32_t taosSetConsoleEcho(bool on);
void setTerminalMode();
int32_t getOldTerminalMode();
void resetTerminalMode();
void taosSetTerminalMode();
int32_t taosGetOldTerminalMode();
void taosResetTerminalMode();
#ifdef __cplusplus
}

1
include/util/taoserror.h
View File

@ -623,6 +623,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_PAR_INVALID_DAYS_VALUE TAOS_DEF_ERROR_CODE(0, 0x2636)
#define TSDB_CODE_PAR_OFFSET_LESS_ZERO TAOS_DEF_ERROR_CODE(0, 0x2637)
#define TSDB_CODE_PAR_SLIMIT_LEAK_PARTITION_BY TAOS_DEF_ERROR_CODE(0, 0x2638)
#define TSDB_CODE_PAR_INVALID_TOPIC_QUERY TAOS_DEF_ERROR_CODE(0, 0x2639)
//planner
#define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700)

2
include/util/tdef.h
View File

@ -76,8 +76,6 @@ extern const int32_t TYPE_BYTES[15];
#define TSDB_DEFAULT_PASS "taosdata"
#endif
#define SHELL_MAX_PASSWORD_LEN 20
#define TSDB_TRUE 1
#define TSDB_FALSE 0
#define TSDB_OK 0

4
include/util/tprocess.h
View File

@ -53,8 +53,8 @@ int32_t taosProcRun(SProcObj *pProc);
void taosProcStop(SProcObj *pProc);
int32_t taosProcPutToChildQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
void *handle, EProcFuncType ftype);
void taosProcRemoveHandle(SProcObj *pProc, void *handle);
void *handle, int64_t handleRef, EProcFuncType ftype);
int64_t taosProcRemoveHandle(SProcObj *pProc, void *handle);
void taosProcCloseHandles(SProcObj *pProc, void (*HandleFp)(void *handle));
void taosProcPutToParentQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
EProcFuncType ftype);

4
source/client/src/clientImpl.c
View File

@ -232,7 +232,9 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra
.mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp),
.pAstRoot = pQuery->pRoot,
.showRewrite = pQuery->showRewrite,
.pTransporter = pRequest->pTscObj->pAppInfo->pTransporter
.pTransporter = pRequest->pTscObj->pAppInfo->pTransporter,
.pMsg = pRequest->msgBuf,
.msgLen = ERROR_MSG_BUF_DEFAULT_SIZE
};
int32_t code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &cxt.pCatalog);
if (TSDB_CODE_SUCCESS == code) {

45
source/common/src/tglobal.c
View File

@ -32,13 +32,9 @@ int32_t tsVersion = 30000000;
int32_t tsStatusInterval = 1; // second
// common
int32_t tsRpcTimer = 300;
int32_t tsRpcMaxTime = 600; // seconds;
bool tsRpcForceTcp = true; // disable this, means query, show command use udp protocol as default
int32_t tsMaxShellConns = 50000;
int32_t tsMaxConnections = 50000;
int32_t tsShellActivityTimer = 3; // second
int32_t tsMaxBinaryDisplayWidth = 30;
bool tsEnableSlaveQuery = true;
bool tsPrintAuth = false;
@ -105,14 +101,6 @@ int32_t tsCompressColData = -1;
*/
int32_t tsCompatibleModel = 1;
// client
int32_t tsMaxWildCardsLen = TSDB_PATTERN_STRING_DEFAULT_LEN;
int32_t tsMaxRegexStringLen = TSDB_REGEX_STRING_DEFAULT_LEN;
// the maximum number of results for projection query on super table that are returned from
// one virtual node, to order according to timestamp
int32_t tsMaxNumOfOrderedResults = 100000;
// 10 ms for sliding time, the value will changed in case of time precision changed
int32_t tsMinSlidingTime = 10;
@ -237,18 +225,20 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char *inputCfgDir, const char *e
char cfgFile[PATH_MAX + 100] = {0};
taosExpandDir(inputCfgDir, cfgDir, PATH_MAX);
snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "taos.cfg", cfgDir);
if (taosIsDir(cfgDir)) {
snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "taos.cfg", cfgDir);
} else {
tstrncpy(cfgFile, cfgDir, sizeof(cfgDir));
}
if (cfgLoad(pCfg, CFG_STYPE_APOLLO_URL, apolloUrl) != 0) {
uError("failed to load from apollo url:%s since %s", apolloUrl, terrstr());
return -1;
}
if (cfgLoad(pCfg, CFG_STYPE_CFG_FILE, cfgDir) != 0) {
if (cfgLoad(pCfg, CFG_STYPE_CFG_FILE, cfgFile) != 0) {
uInfo("cfg file:%s not read since %s", cfgFile, terrstr());
return 0;
}
if (cfgLoad(pCfg, CFG_STYPE_CFG_FILE, cfgFile) != 0) {
uError("failed to load from cfg file:%s since %s", cfgFile, terrstr());
return -1;
}
if (cfgLoad(pCfg, CFG_STYPE_ENV_FILE, envFile) != 0) {
@ -312,19 +302,10 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, 1) != 0) return -1;
if (cfgAddDir(pCfg, "tempDir", tsTempDir, 1) != 0) return -1;
if (cfgAddFloat(pCfg, "minimalTempDirGB", 1.0f, 0.001f, 10000000, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "maxTmrCtrl", tsMaxTmrCtrl, 8, 2048, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "rpcTimer", tsRpcTimer, 100, 3000, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "rpcMaxTime", tsRpcMaxTime, 100, 7200, 1) != 0) return -1;
if (cfgAddBool(pCfg, "rpcForceTcp", tsRpcForceTcp, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "shellActivityTimer", tsShellActivityTimer, 1, 120, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "compressMsgSize", tsCompressMsgSize, -1, 100000000, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "compressColData", tsCompressColData, -1, 100000000, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "maxWildCardsLength", tsMaxWildCardsLen, 0, TSDB_MAX_FIELD_LEN, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "maxRegexStringLen", tsMaxRegexStringLen, 0, TSDB_MAX_FIELD_LEN, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "maxNumOfOrderedRes", tsMaxNumOfOrderedResults, 128, TSDB_MAX_ALLOWED_SQL_LEN, 1) != 0)
return -1;
if (cfgAddBool(pCfg, "keepColumnName", tsKeepOriginalColumnName, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "maxBinaryDisplayWidth", tsMaxBinaryDisplayWidth, 1, 65536, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 3, 1) != 0) return -1;
tsNumOfTaskQueueThreads = tsNumOfCores / 4;
@ -506,18 +487,10 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
return -1;
}
tsMaxTmrCtrl = cfgGetItem(pCfg, "maxTmrCtrl")->i32;
tsRpcTimer = cfgGetItem(pCfg, "rpcTimer")->i32;
tsRpcMaxTime = cfgGetItem(pCfg, "rpcMaxTime")->i32;
tsRpcForceTcp = cfgGetItem(pCfg, "rpcForceTcp")->i32;
tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32;
tsCompressMsgSize = cfgGetItem(pCfg, "compressMsgSize")->i32;
tsCompressColData = cfgGetItem(pCfg, "compressColData")->i32;
tsMaxWildCardsLen = cfgGetItem(pCfg, "maxWildCardsLength")->i32;
tsMaxRegexStringLen = cfgGetItem(pCfg, "maxRegexStringLen")->i32;
tsMaxNumOfOrderedResults = cfgGetItem(pCfg, "maxNumOfOrderedRes")->i32;
tsKeepOriginalColumnName = cfgGetItem(pCfg, "keepColumnName")->bval;
tsMaxBinaryDisplayWidth = cfgGetItem(pCfg, "maxBinaryDisplayWidth")->i32;
tsNumOfTaskQueueThreads = cfgGetItem(pCfg, "numOfTaskQueueThreads")->i32;
tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32;
return 0;
@ -708,6 +681,6 @@ void taosCfgDynamicOptions(const char *option, const char *value) {
if (strcasecmp(option, "resetlog") == 0) {
taosResetLog();
cfgDumpCfg(tsCfg, 1, false);
cfgDumpCfg(tsCfg, 0, false);
}
}

15
source/common/src/tmsg.c
View File

@ -1568,13 +1568,8 @@ int32_t tSerializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pReq
if (tEncodeI32(&encoder, pReq->codeLen) < 0) return -1;
if (tEncodeI64(&encoder, pReq->signature) < 0) return -1;
int32_t codeSize = 0;
if (pReq->pCode != NULL) {
codeSize = strlen(pReq->pCode) + 1;
}
if (tEncodeI32(&encoder, codeSize) < 0) return -1;
if (pReq->pCode != NULL) {
if (tEncodeCStr(&encoder, pReq->pCode) < 0) return -1;
if (tEncodeBinary(&encoder, pReq->pCode, pReq->codeLen) < 0) return -1;
}
int32_t commentSize = 0;
@ -1608,10 +1603,8 @@ int32_t tDeserializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pR
if (tDecodeI32(&decoder, &pReq->codeLen) < 0) return -1;
if (tDecodeI64(&decoder, &pReq->signature) < 0) return -1;
int32_t codeSize = 0;
if (tDecodeI32(&decoder, &codeSize) < 0) return -1;
if (codeSize > 0) {
pReq->pCode = taosMemoryCalloc(1, codeSize);
if (pReq->codeLen > 0) {
pReq->pCode = taosMemoryCalloc(1, pReq->codeLen);
if (pReq->pCode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
@ -1734,7 +1727,7 @@ int32_t tSerializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp *
if (tEncodeI32(&encoder, pInfo->codeSize) < 0) return -1;
if (tEncodeI32(&encoder, pInfo->commentSize) < 0) return -1;
if (pInfo->codeSize) {
if (tEncodeCStr(&encoder, pInfo->pCode) < 0) return -1;
if (tEncodeBinary(&encoder, pInfo->pCode, pInfo->codeSize) < 0) return -1;
}
if (pInfo->commentSize) {
if (tEncodeCStr(&encoder, pInfo->pComment) < 0) return -1;

4
source/dnode/mgmt/exe/dmMain.c
View File

@ -106,7 +106,7 @@ static void dmPrintVersion() {
static void dmDumpCfg() {
SConfig *pCfg = taosGetCfg();
cfgDumpCfg(pCfg, 0, 1);
cfgDumpCfg(pCfg, 0, true);
}
static SDnodeOpt dmGetOpt() {
@ -190,7 +190,7 @@ int main(int argc, char const *argv[]) {
}
if (dmInitLog() != 0) {
printf("failed to start since init log error\n");
dError("failed to start since init log error");
return -1;
}

33
source/dnode/mgmt/implement/src/dmHandle.c
View File

@ -17,7 +17,7 @@
#include "dmImp.h"
static void dmUpdateDnodeCfg(SDnode *pDnode, SDnodeCfg *pCfg) {
if (pDnode->data.dnodeId == 0) {
if (pDnode->data.dnodeId == 0 || pDnode->data.clusterId == 0) {
dInfo("set dnodeId:%d clusterId:%" PRId64, pCfg->dnodeId, pCfg->clusterId);
taosWLockLatch(&pDnode->data.latch);
pDnode->data.dnodeId = pCfg->dnodeId;
@ -57,6 +57,7 @@ void dmSendStatusReq(SDnode *pDnode) {
req.dnodeVer = pDnode->data.dnodeVer;
req.dnodeId = pDnode->data.dnodeId;
req.clusterId = pDnode->data.clusterId;
if (req.clusterId == 0) req.dnodeId = 0;
req.rebootTime = pDnode->data.rebootTime;
req.updateTime = pDnode->data.updateTime;
req.numOfCores = tsNumOfCores;
@ -145,10 +146,10 @@ int32_t dmProcessCreateNodeReq(SDnode *pDnode, EDndNodeType ntype, SNodeMsg *pMs
dError("node:%s, failed to create since %s", pWrapper->name, terrstr());
} else {
dDebug("node:%s, has been created", pWrapper->name);
(void)dmOpenNode(pWrapper);
pWrapper->required = true;
pWrapper->deployed = true;
pWrapper->procType = pDnode->ptype;
(void)dmOpenNode(pWrapper);
}
taosThreadMutexUnlock(&pDnode->mutex);
@ -170,13 +171,13 @@ int32_t dmProcessDropNodeReq(SDnode *pDnode, EDndNodeType ntype, SNodeMsg *pMsg)
dError("node:%s, failed to drop since %s", pWrapper->name, terrstr());
} else {
dDebug("node:%s, has been dropped", pWrapper->name);
pWrapper->required = false;
pWrapper->deployed = false;
}
dmReleaseWrapper(pWrapper);
if (code == 0) {
pWrapper->required = false;
pWrapper->deployed = false;
dmCloseNode(pWrapper);
taosRemoveDir(pWrapper->path);
}
@ -310,6 +311,9 @@ static void dmWatchUdfd(void *args) {
}
static int32_t dmStartUdfd(SDnode *pDnode) {
char dnodeId[8] = {0};
snprintf(dnodeId, sizeof(dnodeId), "%d", pDnode->data.dnodeId);
uv_os_setenv("DNODE_ID", dnodeId);
SUdfdData *pData = &pDnode->udfdData;
if (pData->startCalled) {
dInfo("dnode-mgmt start udfd already called");
@ -319,8 +323,17 @@ static int32_t dmStartUdfd(SDnode *pDnode) {
uv_barrier_init(&pData->barrier, 2);
uv_thread_create(&pData->thread, dmWatchUdfd, pDnode);
uv_barrier_wait(&pData->barrier);
pData->needCleanUp = true;
return pData->spawnErr;
int32_t err = atomic_load_32(&pData->spawnErr);
if (err != 0) {
uv_barrier_destroy(&pData->barrier);
uv_async_send(&pData->stopAsync);
uv_thread_join(&pData->thread);
pData->needCleanUp = false;
dInfo("dnode-mgmt udfd cleaned up after spawn err");
} else {
pData->needCleanUp = true;
}
return err;
}
static int32_t dmStopUdfd(SDnode *pDnode) {
@ -335,7 +348,7 @@ static int32_t dmStopUdfd(SDnode *pDnode) {
uv_barrier_destroy(&pData->barrier);
uv_async_send(&pData->stopAsync);
uv_thread_join(&pData->thread);
dInfo("dnode-mgmt udfd cleaned up");
return 0;
}
@ -370,9 +383,9 @@ static int32_t dmInitMgmt(SMgmtWrapper *pWrapper) {
}
dmReportStartup(pDnode, "dnode-transport", "initialized");
// if (dmStartUdfd(pDnode) != 0) {
// dError("failed to start udfd");
// }
if (dmStartUdfd(pDnode) != 0) {
dError("failed to start udfd");
}
dInfo("dnode-mgmt is initialized");
return 0;

37
source/dnode/mgmt/implement/src/dmTransport.c
View File

@ -59,6 +59,10 @@ static inline int32_t dmBuildMsg(SNodeMsg *pMsg, SRpcMsg *pRpc) {
pMsg->clientIp = connInfo.clientIp;
pMsg->clientPort = connInfo.clientPort;
memcpy(&pMsg->rpcMsg, pRpc, sizeof(SRpcMsg));
if ((pRpc->msgType & 1u)) {
assert(pRpc->refId != 0);
}
// assert(pRpc->handle != NULL && pRpc->refId != 0 && pMsg->rpcMsg.refId != 0);
return 0;
}
@ -67,12 +71,15 @@ static void dmProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSe
SNodeMsg *pMsg = NULL;
NodeMsgFp msgFp = NULL;
uint16_t msgType = pRpc->msgType;
bool needRelease = false;
if (pEpSet && pEpSet->numOfEps > 0 && msgType == TDMT_MND_STATUS_RSP) {
dmSetMnodeEpSet(pWrapper->pDnode, pEpSet);
}
if (dmMarkWrapper(pWrapper) != 0) goto _OVER;
needRelease = true;
if ((msgFp = dmGetMsgFp(pWrapper, pRpc)) == NULL) goto _OVER;
if ((pMsg = taosAllocateQitem(sizeof(SNodeMsg))) == NULL) goto _OVER;
if (dmBuildMsg(pMsg, pRpc) != 0) goto _OVER;
@ -84,7 +91,7 @@ static void dmProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSe
dTrace("msg:%p, is created and put into child queue, type:%s handle:%p user:%s", pMsg, TMSG_INFO(msgType),
pRpc->handle, pMsg->user);
code = taosProcPutToChildQ(pWrapper->procObj, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen, pRpc->handle,
PROC_FUNC_REQ);
pRpc->refId, PROC_FUNC_REQ);
} else {
dTrace("msg:%p, should not processed in child process, handle:%p user:%s", pMsg, pRpc->handle, pMsg->user);
ASSERT(1);
@ -107,7 +114,7 @@ _OVER:
}
}
SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = code};
SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = code, .refId = pRpc->refId};
tmsgSendRsp(&rsp);
}
dTrace("msg:%p, is freed", pMsg);
@ -115,7 +122,9 @@ _OVER:
rpcFreeCont(pRpc->pCont);
}
dmReleaseWrapper(pWrapper);
if (needRelease) {
dmReleaseWrapper(pWrapper);
}
}
static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
@ -131,10 +140,17 @@ static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
return;
}
if (msgType == TDMT_DND_NET_TEST) {
dTrace("net test req will be processed, handle:%p, app:%p", pMsg->handle, pMsg->ahandle);
dmProcessServerStatusReq(pDnode, pMsg);
return;
}
if (pDnode->status != DND_STAT_RUNNING) {
dError("msg:%s ignored since dnode not running, handle:%p app:%p", TMSG_INFO(msgType), pMsg->handle, pMsg->ahandle);
if (isReq) {
SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_APP_NOT_READY, .ahandle = pMsg->ahandle};
SRpcMsg rspMsg = {
.handle = pMsg->handle, .code = TSDB_CODE_APP_NOT_READY, .ahandle = pMsg->ahandle, .refId = pMsg->refId};
rpcSendResponse(&rspMsg);
}
rpcFreeCont(pMsg->pCont);
@ -143,7 +159,8 @@ static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
if (isReq && pMsg->pCont == NULL) {
dError("req:%s not processed since its empty, handle:%p app:%p", TMSG_INFO(msgType), pMsg->handle, pMsg->ahandle);
SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_INVALID_MSG_LEN, .ahandle = pMsg->ahandle};
SRpcMsg rspMsg = {
.handle = pMsg->handle, .code = TSDB_CODE_INVALID_MSG_LEN, .ahandle = pMsg->ahandle, .refId = pMsg->refId};
rpcSendResponse(&rspMsg);
return;
}
@ -151,7 +168,8 @@ static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
if (pWrapper == NULL) {
dError("msg:%s not processed since no handle, handle:%p app:%p", TMSG_INFO(msgType), pMsg->handle, pMsg->ahandle);
if (isReq) {
SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED, .ahandle = pMsg->ahandle};
SRpcMsg rspMsg = {
.handle = pMsg->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED, .ahandle = pMsg->ahandle, .refId = pMsg->refId};
rpcSendResponse(&rspMsg);
}
rpcFreeCont(pMsg->pCont);
@ -170,6 +188,9 @@ static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
}
dTrace("msg:%s will be processed by %s, app:%p", TMSG_INFO(msgType), pWrapper->name, pMsg->ahandle);
if (isReq) {
assert(pMsg->refId != 0);
}
dmProcessRpcMsg(pWrapper, pMsg, pEpSet);
}
@ -317,7 +338,7 @@ static void dmConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int16_t
if (code != 0) {
dError("msg:%p, failed to process since code:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code));
if (pRpc->msgType & 1U) {
SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno};
SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno, .refId = pRpc->refId};
dmSendRsp(pWrapper, &rsp);
}
@ -346,7 +367,7 @@ static void dmConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, int16_t
dmSendRpcReq(pWrapper->pDnode, (SEpSet *)((char *)pMsg + sizeof(SRpcMsg)), pMsg);
break;
case PROC_FUNC_RSP:
taosProcRemoveHandle(pWrapper->procObj, pMsg->handle);
pMsg->refId = taosProcRemoveHandle(pWrapper->procObj, pMsg->handle);
dmSendRpcRsp(pWrapper->pDnode, pMsg);
break;
default:

4
source/dnode/mgmt/implement/src/dmWorker.c
View File

@ -105,7 +105,7 @@ void dmStopMonitorThread(SDnode *pDnode) {
}
static void dmProcessMgmtQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
SDnode *pDnode = pInfo->ahandle;
SDnode * pDnode = pInfo->ahandle;
SRpcMsg *pRpc = &pMsg->rpcMsg;
int32_t code = -1;
dTrace("msg:%p, will be processed in dnode-mgmt queue", pMsg);
@ -150,7 +150,7 @@ static void dmProcessMgmtQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
if (pRpc->msgType & 1u) {
if (code != 0) code = terrno;
SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = code};
SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = code, .refId = pRpc->refId};
rpcSendResponse(&rsp);
}

1
source/dnode/mgmt/interface/inc/dmInt.h
View File

@ -37,6 +37,7 @@ void dmSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgF
void dmReportStartup(SDnode *pDnode, const char *pName, const char *pDesc);
void dmReportStartupByWrapper(SMgmtWrapper *pWrapper, const char *pName, const char *pDesc);
void dmProcessServerStatusReq(SDnode *pDnode, SRpcMsg *pMsg);
void dmProcessNettestReq(SDnode *pDnode, SRpcMsg *pMsg);
void dmGetMonitorSysInfo(SMonSysInfo *pInfo);
// dmFile.c

10
source/dnode/mgmt/interface/src/dmInt.c
View File

@ -171,13 +171,21 @@ static void dmGetServerStatus(SDnode *pDnode, SServerStatusRsp *pStatus) {
}
}
void dmProcessNettestReq(SDnode *pDnode, SRpcMsg *pRpc) {
dDebug("net test req is received");
SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = 0};
rsp.pCont = rpcMallocCont(pRpc->contLen);
rsp.contLen = pRpc->contLen;
rpcSendResponse(&rsp);
}
void dmProcessServerStatusReq(SDnode *pDnode, SRpcMsg *pReq) {
dDebug("server status req is received");
SServerStatusRsp statusRsp = {0};
dmGetServerStatus(pDnode, &statusRsp);
SRpcMsg rspMsg = {.handle = pReq->handle, .ahandle = pReq->ahandle};
SRpcMsg rspMsg = {.handle = pReq->handle, .ahandle = pReq->ahandle, .refId = pReq->refId};
int32_t rspLen = tSerializeSServerStatusRsp(NULL, 0, &statusRsp);
if (rspLen < 0) {
rspMsg.code = TSDB_CODE_OUT_OF_MEMORY;

2
source/dnode/mgmt/mgmt_bnode/src/bmHandle.c
View File

@ -53,7 +53,7 @@ int32_t bmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
return -1;
}
if (createReq.dnodeId != pDnode->data.dnodeId) {
if (pDnode->data.dnodeId != 0 && createReq.dnodeId != pDnode->data.dnodeId) {
terrno = TSDB_CODE_INVALID_OPTION;
dError("failed to create bnode since %s, input:%d cur:%d", terrstr(), createReq.dnodeId, pDnode->data.dnodeId);
return -1;

8
source/dnode/mgmt/mgmt_bnode/src/bmWorker.c
View File

@ -17,7 +17,8 @@
#include "bmInt.h"
static void bmSendErrorRsp(SNodeMsg *pMsg, int32_t code) {
SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code};
SRpcMsg rpcRsp = {
.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code, .refId = pMsg->rpcMsg.refId};
tmsgSendRsp(&rpcRsp);
dTrace("msg:%p, is freed", pMsg);
@ -38,6 +39,7 @@ static void bmSendErrorRsps(STaosQall *qall, int32_t numOfMsgs, int32_t code) {
static inline void bmSendRsp(SNodeMsg *pMsg, int32_t code) {
SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle,
.ahandle = pMsg->rpcMsg.ahandle,
.refId = pMsg->rpcMsg.refId,
.code = code,
.pCont = pMsg->pRsp,
.contLen = pMsg->rspLen};
@ -101,7 +103,7 @@ static void bmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
}
int32_t bmProcessWriteMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
SBnodeMgmt *pMgmt = pWrapper->pMgmt;
SBnodeMgmt * pMgmt = pWrapper->pMgmt;
SMultiWorker *pWorker = &pMgmt->writeWorker;
dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);
@ -110,7 +112,7 @@ int32_t bmProcessWriteMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
}
int32_t bmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
SBnodeMgmt *pMgmt = pWrapper->pMgmt;
SBnodeMgmt * pMgmt = pWrapper->pMgmt;
SSingleWorker *pWorker = &pMgmt->monitorWorker;
dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);

4
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -134,9 +134,9 @@ int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) {
return -1;
}
if (alterReq.dnodeId != pDnode->data.dnodeId) {
if (pDnode->data.dnodeId != 0 && alterReq.dnodeId != pDnode->data.dnodeId) {
terrno = TSDB_CODE_INVALID_OPTION;
dError("failed to alter mnode since %s, dnodeId:%d input:%d", terrstr(), pDnode->data.dnodeId, alterReq.dnodeId);
dError("failed to alter mnode since %s, input:%d cur:%d", terrstr(), alterReq.dnodeId, pDnode->data.dnodeId);
return -1;
} else {
return mmAlter(pMgmt, &alterReq);

4
source/dnode/mgmt/mgmt_mnode/src/mmInt.c
View File

@ -161,9 +161,9 @@ static int32_t mmOpen(SMgmtWrapper *pWrapper) {
SMnodeOpt option = {0};
if (!deployed) {
dInfo("mnode start to deploy");
if (pWrapper->procType == DND_PROC_CHILD) {
// if (pWrapper->procType == DND_PROC_CHILD) {
pWrapper->pDnode->data.dnodeId = 1;
}
// }
mmBuildOptionForDeploy(pMgmt, &option);
} else {
dInfo("mnode start to open");

1
source/dnode/mgmt/mgmt_mnode/src/mmWorker.c
View File

@ -19,6 +19,7 @@
static inline void mmSendRsp(SNodeMsg *pMsg, int32_t code) {
SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle,
.ahandle = pMsg->rpcMsg.ahandle,
.refId = pMsg->rpcMsg.refId,
.code = code,
.pCont = pMsg->pRsp,
.contLen = pMsg->rspLen};

1
source/dnode/mgmt/mgmt_qnode/src/qmWorker.c
View File

@ -19,6 +19,7 @@
static inline void qmSendRsp(SNodeMsg *pMsg, int32_t code) {
SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle,
.ahandle = pMsg->rpcMsg.ahandle,
.refId = pMsg->rpcMsg.refId,
.code = code,
.pCont = pMsg->pRsp,
.contLen = pMsg->rspLen};

9
source/dnode/mgmt/mgmt_snode/src/smWorker.c
View File

@ -19,6 +19,7 @@
static inline void smSendRsp(SNodeMsg *pMsg, int32_t code) {
SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle,
.ahandle = pMsg->rpcMsg.ahandle,
.refId = pMsg->rpcMsg.refId,
.code = code,
.pCont = pMsg->pRsp,
.contLen = pMsg->rspLen};
@ -149,7 +150,7 @@ static FORCE_INLINE int32_t smGetSWTypeFromMsg(SRpcMsg *pMsg) {
}
int32_t smProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
SSnodeMgmt *pMgmt = pWrapper->pMgmt;
SSnodeMgmt * pMgmt = pWrapper->pMgmt;
SMultiWorker *pWorker = taosArrayGetP(pMgmt->uniqueWorkers, 0);
if (pWorker == NULL) {
terrno = TSDB_CODE_INVALID_MSG;
@ -162,7 +163,7 @@ int32_t smProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
}
int32_t smProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
SSnodeMgmt *pMgmt = pWrapper->pMgmt;
SSnodeMgmt * pMgmt = pWrapper->pMgmt;
SSingleWorker *pWorker = &pMgmt->monitorWorker;
dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);
@ -171,7 +172,7 @@ int32_t smProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
}
int32_t smProcessUniqueMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
SSnodeMgmt *pMgmt = pWrapper->pMgmt;
SSnodeMgmt * pMgmt = pWrapper->pMgmt;
int32_t index = smGetSWIdFromMsg(&pMsg->rpcMsg);
SMultiWorker *pWorker = taosArrayGetP(pMgmt->uniqueWorkers, index);
if (pWorker == NULL) {
@ -185,7 +186,7 @@ int32_t smProcessUniqueMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
}
int32_t smProcessSharedMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
SSnodeMgmt *pMgmt = pWrapper->pMgmt;
SSnodeMgmt * pMgmt = pWrapper->pMgmt;
SSingleWorker *pWorker = &pMgmt->sharedWorker;
dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);

23
source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
View File

@ -23,6 +23,7 @@
static inline void vmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) {
SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle,
.ahandle = pMsg->rpcMsg.ahandle,
.refId = pMsg->rpcMsg.refId,
.code = code,
.pCont = pMsg->pRsp,
.contLen = pMsg->rspLen};
@ -126,6 +127,7 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
rsp.code = 0;
rsp.handle = pRpc->handle;
rsp.ahandle = pRpc->ahandle;
rsp.refId = pRpc->refId;
int32_t code = vnodeProcessWriteReq(pVnode->pImpl, pRpc, version++, &rsp);
tmsgSendRsp(&rsp);
@ -134,13 +136,14 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
// sync integration response
for (int i = 0; i < taosArrayGetSize(pArray); i++) {
SNodeMsg *pMsg;
SRpcMsg *pRpc;
SRpcMsg * pRpc;
pMsg = *(SNodeMsg **)taosArrayGet(pArray, i);
pRpc = &pMsg->rpcMsg;
rsp.ahandle = pRpc->ahandle;
rsp.handle = pRpc->handle;
rsp.refId = pRpc->refId;
rsp.pCont = NULL;
rsp.contLen = 0;
@ -172,11 +175,9 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
SVnodeObj *pVnode = pInfo->ahandle;
SNodeMsg *pMsg = NULL;
SNodeMsg * pMsg = NULL;
SRpcMsg rsp;
// static int64_t version = 0;
for (int32_t i = 0; i < numOfMsgs; ++i) {
#if 1
// sync integration
@ -208,6 +209,7 @@ static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
if (pMsg->rpcMsg.handle != NULL && pMsg->rpcMsg.ahandle != NULL) {
rsp.ahandle = pMsg->rpcMsg.ahandle;
rsp.handle = pMsg->rpcMsg.handle;
rsp.refId = pMsg->rpcMsg.refId;
tmsgSendRsp(&rsp);
}
#endif
@ -216,7 +218,7 @@ static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
SVnodeObj *pVnode = pInfo->ahandle;
SNodeMsg *pMsg = NULL;
SNodeMsg * pMsg = NULL;
for (int32_t i = 0; i < numOfMsgs; ++i) {
taosGetQitem(qall, (void **)&pMsg);
@ -229,7 +231,7 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf
static void vmProcessMergeQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
SVnodeObj *pVnode = pInfo->ahandle;
SNodeMsg *pMsg = NULL;
SNodeMsg * pMsg = NULL;
for (int32_t i = 0; i < numOfMsgs; ++i) {
taosGetQitem(qall, (void **)&pMsg);
@ -246,7 +248,7 @@ static void vmProcessMergeQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
}
static int32_t vmPutNodeMsgToQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg, EQueueType qtype) {
SRpcMsg *pRpc = &pMsg->rpcMsg;
SRpcMsg * pRpc = &pMsg->rpcMsg;
SMsgHead *pHead = pRpc->pCont;
pHead->contLen = ntohl(pHead->contLen);
pHead->vgId = ntohl(pHead->vgId);
@ -315,7 +317,7 @@ int32_t vmProcessMergeMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
}
int32_t vmProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
SVnodesMgmt *pMgmt = pWrapper->pMgmt;
SVnodesMgmt * pMgmt = pWrapper->pMgmt;
SSingleWorker *pWorker = &pMgmt->mgmtWorker;
dTrace("msg:%p, will be written to vnode-mgmt queue, worker:%s", pMsg, pWorker->name);
taosWriteQitem(pWorker->queue, pMsg);
@ -323,7 +325,7 @@ int32_t vmProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
}
int32_t vmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
SVnodesMgmt *pMgmt = pWrapper->pMgmt;
SVnodesMgmt * pMgmt = pWrapper->pMgmt;
SSingleWorker *pWorker = &pMgmt->monitorWorker;
dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);
@ -333,7 +335,7 @@ int32_t vmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
static int32_t vmPutRpcMsgToQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, EQueueType qtype) {
SVnodesMgmt *pMgmt = pWrapper->pMgmt;
SMsgHead *pHead = pRpc->pCont;
SMsgHead * pHead = pRpc->pCont;
SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId);
if (pVnode == NULL) return -1;
@ -346,6 +348,7 @@ static int32_t vmPutRpcMsgToQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, EQueueT
} else {
dTrace("msg:%p, is created, type:%s", pMsg, TMSG_INFO(pRpc->msgType));
pMsg->rpcMsg = *pRpc;
// if (pMsg->rpcMsg.handle != NULL) assert(pMsg->rpcMsg.refId != 0);
switch (qtype) {
case QUERY_QUEUE:
dTrace("msg:%p, will be put into vnode-query queue", pMsg);

1
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -135,6 +135,7 @@ typedef struct {
int32_t failedTimes;
void* rpcHandle;
void* rpcAHandle;
int64_t rpcRefId;
void* rpcRsp;
int32_t rpcRspLen;
SArray* redoLogs;

8
source/dnode/mnode/impl/src/mndFunc.c
View File

@ -309,10 +309,10 @@ static int32_t mndProcessCreateFuncReq(SNodeMsg *pReq) {
goto _OVER;
}
if (createReq.pCode[0] == 0) {
terrno = TSDB_CODE_MND_INVALID_FUNC_CODE;
goto _OVER;
}
if (createReq.codeLen <= 1) {
terrno = TSDB_CODE_MND_INVALID_FUNC_CODE;
goto _OVER;
}
if (createReq.bufSize <= 0 || createReq.bufSize > TSDB_FUNC_BUF_SIZE) {
terrno = TSDB_CODE_MND_INVALID_FUNC_BUFSIZE;

27
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -193,9 +193,9 @@ TRANS_ENCODE_OVER:
static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
SSdbRow *pRow = NULL;
STrans *pTrans = NULL;
char *pData = NULL;
SSdbRow * pRow = NULL;
STrans * pTrans = NULL;
char * pData = NULL;
int32_t dataLen = 0;
int8_t sver = 0;
int32_t redoLogNum = 0;
@ -456,7 +456,7 @@ static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOld, STrans *pNew) {
}
static STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId) {
SSdb *pSdb = pMnode->pSdb;
SSdb * pSdb = pMnode->pSdb;
STrans *pTrans = sdbAcquire(pSdb, SDB_TRANS, &transId);
if (pTrans == NULL) {
terrno = TSDB_CODE_MND_TRANS_NOT_EXIST;
@ -484,6 +484,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnType type, const S
pTrans->createdTime = taosGetTimestampMs();
pTrans->rpcHandle = pReq->handle;
pTrans->rpcAHandle = pReq->ahandle;
pTrans->rpcRefId = pReq->refId;
pTrans->redoLogs = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(void *));
pTrans->undoLogs = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(void *));
pTrans->commitLogs = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(void *));
@ -625,7 +626,7 @@ static int32_t mndCheckTransCanBeStartedInParallel(SMnode *pMnode, STrans *pNewT
if (mndIsBasicTrans(pNewTrans)) return 0;
STrans *pTrans = NULL;
void *pIter = NULL;
void * pIter = NULL;
int32_t code = 0;
while (1) {
@ -703,6 +704,7 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) {
pNew->rpcHandle = pTrans->rpcHandle;
pNew->rpcAHandle = pTrans->rpcAHandle;
pNew->rpcRefId = pTrans->rpcRefId;
pNew->rpcRsp = pTrans->rpcRsp;
pNew->rpcRspLen = pTrans->rpcRspLen;
pTrans->rpcRsp = NULL;
@ -767,6 +769,7 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) {
SRpcMsg rspMsg = {.handle = pTrans->rpcHandle,
.code = pTrans->code,
.ahandle = pTrans->rpcAHandle,
.refId = pTrans->rpcRefId,
.pCont = rpcCont,
.contLen = pTrans->rpcRspLen};
tmsgSendRsp(&rspMsg);
@ -827,7 +830,7 @@ HANDLE_ACTION_RSP_OVER:
}
static int32_t mndTransExecuteLogs(SMnode *pMnode, SArray *pArray) {
SSdb *pSdb = pMnode->pSdb;
SSdb * pSdb = pMnode->pSdb;
int32_t arraySize = taosArrayGetSize(pArray);
if (arraySize == 0) return 0;
@ -1202,11 +1205,11 @@ static int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans) {
}
static int32_t mndProcessKillTransReq(SNodeMsg *pReq) {
SMnode *pMnode = pReq->pNode;
SMnode * pMnode = pReq->pNode;
SKillTransReq killReq = {0};
int32_t code = -1;
SUserObj *pUser = NULL;
STrans *pTrans = NULL;
SUserObj * pUser = NULL;
STrans * pTrans = NULL;
if (tDeserializeSKillTransReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &killReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
@ -1246,7 +1249,7 @@ KILL_OVER:
void mndTransPullup(SMnode *pMnode) {
STrans *pTrans = NULL;
void *pIter = NULL;
void * pIter = NULL;
while (1) {
pIter = sdbFetch(pMnode->pSdb, SDB_TRANS, pIter, (void **)&pTrans);
@ -1261,11 +1264,11 @@ void mndTransPullup(SMnode *pMnode) {
static int32_t mndRetrieveTrans(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
SMnode *pMnode = pReq->pNode;
SSdb *pSdb = pMnode->pSdb;
SSdb * pSdb = pMnode->pSdb;
int32_t numOfRows = 0;
STrans *pTrans = NULL;
int32_t cols = 0;
char *pWrite;
char * pWrite;
while (numOfRows < rows) {
pShow->pIter = sdbFetch(pSdb, SDB_TRANS, pShow->pIter, (void **)&pTrans);

6
source/dnode/mnode/impl/src/mnode.c
View File

@ -385,7 +385,11 @@ int32_t mndProcessMsg(SNodeMsg *pMsg) {
terrno = code;
mTrace("msg:%p, in progress, app:%p", pMsg, ahandle);
} else if (code != 0) {
mError("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle);
if (terrno != TSDB_CODE_OPS_NOT_SUPPORT) {
mError("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle);
} else {
mTrace("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle);
}
} else {
mTrace("msg:%p, is processed, app:%p", pMsg, ahandle);
}

88
source/dnode/mnode/impl/test/func/func.cpp
View File

@ -22,17 +22,16 @@ class MndTestFunc : public ::testing::Test {
void SetUp() override {}
void TearDown() override {}
void SetCode(SCreateFuncReq* pReq, const char* pCode);
void SetCode(SCreateFuncReq* pReq, const char* pCode, int32_t size);
void SetComment(SCreateFuncReq* pReq, const char* pComment);
};
Testbase MndTestFunc::test;
void MndTestFunc::SetCode(SCreateFuncReq* pReq, const char* pCode) {
int32_t len = strlen(pCode);
pReq->pCode = (char*)taosMemoryCalloc(1, len + 1);
strcpy(pReq->pCode, pCode);
pReq->codeLen = len;
void MndTestFunc::SetCode(SCreateFuncReq *pReq, const char *pCode, int32_t size) {
pReq->pCode = (char*)taosMemoryMalloc(size);
memcpy(pReq->pCode, pCode, size);
pReq->codeLen = size;
}
void MndTestFunc::SetComment(SCreateFuncReq* pReq, const char* pComment) {
@ -79,7 +78,7 @@ TEST_F(MndTestFunc, 02_Create_Func) {
{
SCreateFuncReq createReq = {0};
strcpy(createReq.name, "f1");
SetCode(&createReq, "");
SetCode(&createReq, "", 1);
SetComment(&createReq, "comment1");
int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq);
@ -95,7 +94,7 @@ TEST_F(MndTestFunc, 02_Create_Func) {
{
SCreateFuncReq createReq = {0};
strcpy(createReq.name, "f1");
SetCode(&createReq, "code1");
SetCode(&createReq, "code1", 6);
SetComment(&createReq, "comment1");
int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq);
@ -111,7 +110,7 @@ TEST_F(MndTestFunc, 02_Create_Func) {
{
SCreateFuncReq createReq = {0};
strcpy(createReq.name, "f1");
SetCode(&createReq, "code1");
SetCode(&createReq, "code1", 6);
SetComment(&createReq, "comment1");
createReq.bufSize = TSDB_FUNC_BUF_SIZE + 1;
@ -128,7 +127,7 @@ TEST_F(MndTestFunc, 02_Create_Func) {
for (int32_t i = 0; i < 3; ++i) {
SCreateFuncReq createReq = {0};
strcpy(createReq.name, "f1");
SetCode(&createReq, "code1");
SetCode(&createReq, "code1", 6);
SetComment(&createReq, "comment1");
createReq.bufSize = TSDB_FUNC_BUF_SIZE + 1;
createReq.igExists = 0;
@ -253,7 +252,7 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) {
createReq.outputLen = 24;
createReq.bufSize = 6;
createReq.signature = 18;
SetCode(&createReq, "code2");
SetCode(&createReq, "code2", 6);
SetComment(&createReq, "comment2");
int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq);
@ -439,3 +438,70 @@ TEST_F(MndTestFunc, 04_Drop_Func) {
test.SendShowReq(TSDB_MGMT_TABLE_FUNC, "user_functions", "");
EXPECT_EQ(test.GetShowRows(), 1);
}
TEST_F(MndTestFunc, 05_Actual_code) {
{
SCreateFuncReq createReq = {0};
strcpy(createReq.name, "udf1");
char code[300] = {0};
for (int32_t i = 0; i < sizeof(code); ++i) {
code[i] = (i) % 20;
}
SetCode(&createReq, code, 300);
SetComment(&createReq, "comment1");
createReq.bufSize = 8;
createReq.igExists = 0;
createReq.funcType = 1;
createReq.scriptType = 2;
createReq.outputType = TSDB_DATA_TYPE_SMALLINT;
createReq.outputLen = 12;
createReq.bufSize = 4;
createReq.signature = 5;
int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq);
void* pReq = rpcMallocCont(contLen);
tSerializeSCreateFuncReq(pReq, contLen, &createReq);
tFreeSCreateFuncReq(&createReq);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0);
}
{
SRetrieveFuncReq retrieveReq = {0};
retrieveReq.numOfFuncs = 1;
retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN);
taosArrayPush(retrieveReq.pFuncNames, "udf1");
int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq);
void* pReq = rpcMallocCont(contLen);
tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq);
tFreeSRetrieveFuncReq(&retrieveReq);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0);
SRetrieveFuncRsp retrieveRsp = {0};
tDeserializeSRetrieveFuncRsp(pRsp->pCont, pRsp->contLen, &retrieveRsp);
EXPECT_EQ(retrieveRsp.numOfFuncs, 1);
EXPECT_EQ(retrieveRsp.numOfFuncs, (int32_t)taosArrayGetSize(retrieveRsp.pFuncInfos));
SFuncInfo* pFuncInfo = (SFuncInfo*)taosArrayGet(retrieveRsp.pFuncInfos, 0);
EXPECT_STREQ(pFuncInfo->name, "udf1");
EXPECT_EQ(pFuncInfo->funcType, 1);
EXPECT_EQ(pFuncInfo->scriptType, 2);
EXPECT_EQ(pFuncInfo->outputType, TSDB_DATA_TYPE_SMALLINT);
EXPECT_EQ(pFuncInfo->outputLen, 12);
EXPECT_EQ(pFuncInfo->bufSize, 4);
EXPECT_EQ(pFuncInfo->signature, 5);
EXPECT_STREQ("comment1", pFuncInfo->pComment);
for (int32_t i = 0; i < 300; ++i) {
EXPECT_EQ(pFuncInfo->pCode[i], (i) % 20);
}
tFreeSRetrieveFuncRsp(&retrieveRsp);
}
}

1
source/dnode/vnode/src/tsdb/tsdbSma.c
View File

@ -1510,7 +1510,6 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, char *pData, int64_t indexUid,
}
STSma *pTSma = pItem->pSma;
#endif
STSmaReadH tReadH = {0};

11
source/dnode/vnode/src/vnd/vnodeQuery.c
View File

@ -22,21 +22,21 @@ int vnodeQueryOpen(SVnode *pVnode) {
void vnodeQueryClose(SVnode *pVnode) { qWorkerDestroy((void **)&pVnode->pQuery); }
int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) {
STbCfg *pTbCfg = NULL;
STbCfg *pStbCfg = NULL;
STbCfg * pTbCfg = NULL;
STbCfg * pStbCfg = NULL;
tb_uid_t uid;
int32_t nCols;
int32_t nTagCols;
SSchemaWrapper *pSW = NULL;
STableMetaRsp *pTbMetaMsg = NULL;
STableMetaRsp * pTbMetaMsg = NULL;
STableMetaRsp metaRsp = {0};
SSchema *pTagSchema;
SSchema * pTagSchema;
SRpcMsg rpcMsg;
int msgLen = 0;
int32_t code = 0;
char tableFName[TSDB_TABLE_FNAME_LEN];
int32_t rspLen = 0;
void *pRsp = NULL;
void * pRsp = NULL;
STableInfoReq infoReq = {0};
if (tDeserializeSTableInfoReq(pMsg->pCont, pMsg->contLen, &infoReq) != 0) {
@ -142,6 +142,7 @@ _exit:
rpcMsg.handle = pMsg->handle;
rpcMsg.ahandle = pMsg->ahandle;
rpcMsg.refId = pMsg->refId;
rpcMsg.pCont = pRsp;
rpcMsg.contLen = rspLen;
rpcMsg.code = code;

3
source/dnode/vnode/src/vnd/vnodeSync.c
View File

@ -113,7 +113,7 @@ void vnodeSyncCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cb
pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), beginIndex);
syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
SVnode *pVnode = (SVnode *)(pFsm->data);
SVnode * pVnode = (SVnode *)(pFsm->data);
SyncApplyMsg *pSyncApplyMsg = syncApplyMsgBuild2(pMsg, pVnode->config.vgId, &cbMeta);
SRpcMsg applyMsg;
syncApplyMsg2RpcMsg(pSyncApplyMsg, &applyMsg);
@ -133,6 +133,7 @@ void vnodeSyncCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cb
if (ret == 1 && cbMeta.state == TAOS_SYNC_STATE_LEADER) {
applyMsg.handle = saveRpcMsg.handle;
applyMsg.ahandle = saveRpcMsg.ahandle;
applyMsg.refId = saveRpcMsg.refId;
} else {
applyMsg.handle = NULL;
applyMsg.ahandle = NULL;

27
source/libs/command/src/explain.c
View File

@ -344,11 +344,6 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pTagScanNode->node.pOutputDataBlockDesc->totalRowSize);
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_LOOPS_FORMAT, pTagScanNode->count);
if (pTagScanNode->reverse) {
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_REVERSE_FORMAT, pTagScanNode->reverse);
}
EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT);
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
@ -361,10 +356,6 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTagScanNode->order));
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
if (pResNode->pExecInfo) {
QRY_ERR_RET(qExplainBufAppendVerboseExecInfo(pResNode->pExecInfo, tbuf, &tlen));
if (tlen) {
@ -388,11 +379,6 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pTblScanNode->scan.node.pOutputDataBlockDesc->totalRowSize);
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_LOOPS_FORMAT, pTblScanNode->scan.count);
if (pTblScanNode->scan.reverse) {
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_REVERSE_FORMAT, pTblScanNode->scan.reverse);
}
EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT);
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
@ -405,10 +391,6 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTblScanNode->scan.order));
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIMERANGE_FORMAT, pTblScanNode->scanRange.skey, pTblScanNode->scanRange.ekey);
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
@ -434,11 +416,6 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pSTblScanNode->scan.node.pOutputDataBlockDesc->totalRowSize);
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_LOOPS_FORMAT, pSTblScanNode->scan.count);
if (pSTblScanNode->scan.reverse) {
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_REVERSE_FORMAT, pSTblScanNode->scan.reverse);
}
EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT);
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
@ -451,10 +428,6 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pSTblScanNode->scan.order));
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
if (pSTblScanNode->scan.node.pConditions) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);
QRY_ERR_RET(nodesNodeToSQL(pSTblScanNode->scan.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));

1147
source/libs/executor/src/executorimpl.c
View File

File diff suppressed because it is too large Load Diff

5
source/libs/function/inc/builtinsimpl.h
View File

@ -40,6 +40,11 @@ bool getMinmaxFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
int32_t minFunction(SqlFunctionCtx* pCtx);
int32_t maxFunction(SqlFunctionCtx *pCtx);
bool getAvgFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool avgFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
int32_t avgFunction(SqlFunctionCtx* pCtx);
int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId);
bool getStddevFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool stddevFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
int32_t stddevFunction(SqlFunctionCtx* pCtx);

13
source/libs/function/inc/tudf.h
View File

@ -29,29 +29,32 @@ extern "C" {
#define UDF_LISTEN_PIPE_NAME_LEN 32
#define UDF_LISTEN_PIPE_NAME_PREFIX "udfd.sock."
#define UDF_DNODE_ID_ENV_NAME "DNODE_ID"
//======================================================================================
//begin API to taosd and qworker
enum {
UDFC_CODE_STOPPING = -1,
UDFC_CODE_PIPE_READ_ERR = -3,
UDFC_CODE_PIPE_READ_ERR = -2,
UDFC_CODE_CONNECT_PIPE_ERR = -3,
UDFC_CODE_LOAD_UDF_FAILURE = -4,
UDFC_CODE_INVALID_STATE = -5
};
typedef void *UdfcHandle;
typedef void *UdfcFuncHandle;
/**
* create udfd proxy, called once in process that call setupUdf/callUdfxxx/teardownUdf
* @return error code
*/
int32_t udfcOpen(int32_t dnodeId, UdfcHandle* proxyHandle);
int32_t udfcOpen();
/**
* destroy udfd proxy
* @return error code
*/
int32_t udfcClose(UdfcHandle proxyhandle);
int32_t udfcClose();
/**
@ -60,7 +63,7 @@ int32_t udfcClose(UdfcHandle proxyhandle);
* @param handle, out
* @return error code
*/
int32_t setupUdf(UdfcHandle proxyHandle, char udfName[], SEpSet *epSet, UdfcFuncHandle *handle);
int32_t setupUdf(char udfName[], UdfcFuncHandle *handle);
typedef struct SUdfColumnMeta {
int16_t type;

2
source/libs/function/inc/tudfInt.h
View File

@ -39,7 +39,6 @@ enum {
typedef struct SUdfSetupRequest {
char udfName[TSDB_FUNC_NAME_LEN];
SEpSet epSet;
} SUdfSetupRequest;
typedef struct SUdfSetupResponse {
@ -112,6 +111,7 @@ void freeUdfDataDataBlock(SUdfDataBlock *block);
int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlock);
int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block);
int32_t getUdfdPipeName(char* pipeName, int32_t size);
#ifdef __cplusplus
}
#endif

14
source/libs/function/src/builtins.c
View File

@ -104,7 +104,7 @@ static int32_t translateCount(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
if (1 != LIST_LENGTH(pFunc->pParameterList)) {
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
}
pFunc->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_BIGINT};
pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT};
return TSDB_CODE_SUCCESS;
}
@ -355,7 +355,7 @@ static int32_t translateToIso8601(SFunctionNode* pFunc, char* pErrBuf, int32_t l
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
pFunc->node.resType = (SDataType) { .bytes = 64, .type = TSDB_DATA_TYPE_BINARY};
pFunc->node.resType = (SDataType) { .bytes = 24, .type = TSDB_DATA_TYPE_BINARY};
return TSDB_CODE_SUCCESS;
}
@ -479,6 +479,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.processFunc = stddevFunction,
.finalizeFunc = stddevFinalize
},
{
.name = "avg",
.type = FUNCTION_TYPE_AVG,
.classification = FUNC_MGT_AGG_FUNC,
.translateFunc = translateInNumOutDou,
.getEnvFunc = getAvgFuncEnv,
.initFunc = avgFunctionSetup,
.processFunc = avgFunction,
.finalizeFunc = avgFinalize
},
{
.name = "percentile",
.type = FUNCTION_TYPE_PERCENTILE,

350
source/libs/function/src/builtinsimpl.c
View File

@ -20,6 +20,59 @@
#include "tdatablock.h"
#include "tpercentile.h"
typedef struct SSumRes {
union {
int64_t isum;
uint64_t usum;
double dsum;
};
} SSumRes;
typedef struct SAvgRes {
double result;
SSumRes sum;
int64_t count;
} SAvgRes;
typedef struct STopBotResItem {
SVariant v;
uint64_t uid; // it is a table uid, used to extract tag data during building of the final result for the tag data
struct {
int32_t pageId;
int32_t offset;
} tuplePos; // tuple data of this chosen row
} STopBotResItem;
typedef struct STopBotRes {
int32_t pageId;
// int32_t num;
STopBotResItem *pItems;
} STopBotRes;
typedef struct SStddevRes {
double result;
int64_t count;
union {double quadraticDSum; int64_t quadraticISum;};
union {double dsum; int64_t isum;};
} SStddevRes;
typedef struct SPercentileInfo {
double result;
tMemBucket *pMemBucket;
int32_t stage;
double minval;
double maxval;
int64_t numOfElems;
} SPercentileInfo;
typedef struct SDiffInfo {
bool hasPrev;
bool includeNull;
bool ignoreNegative;
bool firstOutput;
union { int64_t i64; double d64;} prev;
} SDiffInfo;
#define SET_VAL(_info, numOfElem, res) \
do { \
if ((numOfElem) <= 0) { \
@ -28,13 +81,50 @@
(_info)->numOfRes = (res); \
} while (0)
typedef struct SSumRes {
union {
int64_t isum;
uint64_t usum;
double dsum;
};
} SSumRes;
#define GET_TS_LIST(x) ((TSKEY*)((x)->ptsList))
#define GET_TS_DATA(x, y) (GET_TS_LIST(x)[(y)])
#define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx) \
do { \
for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \
SqlFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \
__ctx->fpSet.process(__ctx); \
} \
} while (0);
#define DO_UPDATE_SUBSID_RES(ctx, ts) \
do { \
for (int32_t _i = 0; _i < (ctx)->subsidiaryRes.numOfCols; ++_i) { \
SqlFunctionCtx *__ctx = (ctx)->subsidiaryRes.pCtx[_i]; \
if (__ctx->functionId == FUNCTION_TS_DUMMY) { \
__ctx->tag.i = (ts); \
__ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \
} \
__ctx->fpSet.process(__ctx); \
} \
} while (0)
#define UPDATE_DATA(ctx, left, right, num, sign, _ts) \
do { \
if (((left) < (right)) ^ (sign)) { \
(left) = (right); \
DO_UPDATE_SUBSID_RES(ctx, _ts); \
(num) += 1; \
} \
} while (0)
#define LOOPCHECK_N(val, _col, ctx, _t, _nrow, _start, sign, num) \
do { \
_t *d = (_t *)((_col)->pData); \
for (int32_t i = (_start); i < (_nrow) + (_start); ++i) { \
if (((_col)->hasNull) && colDataIsNull_f((_col)->nullbitmap, i)) { \
continue; \
} \
TSKEY ts = (ctx)->ptsList != NULL ? GET_TS_DATA(ctx, i) : 0; \
UPDATE_DATA(ctx, val, d[i], num, sign, ts); \
} \
} while (0)
bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) {
if (pResultInfo->initialized) {
@ -135,7 +225,7 @@ int32_t sumFunction(SqlFunctionCtx *pCtx) {
int32_t type = pInput->pData[0]->info.type;
SSumRes* pSumRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
if (pInput->colDataAggIsSet) {
numOfElem = pInput->numOfRows - pAgg->numOfNull;
ASSERT(numOfElem >= 0);
@ -190,6 +280,145 @@ bool getSumFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
return true;
}
bool getAvgFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
pEnv->calcMemSize = sizeof(double);
return true;
}
bool avgFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) {
if (!functionSetup(pCtx, pResultInfo)) {
return false;
}
SAvgRes* pRes = GET_ROWCELL_INTERBUF(pResultInfo);
memset(pRes, 0, sizeof(SAvgRes));
return true;
}
int32_t avgFunction(SqlFunctionCtx* pCtx) {
int32_t numOfElem = 0;
// Only the pre-computing information loaded and actual data does not loaded
SInputColumnInfoData* pInput = &pCtx->input;
int32_t type = pInput->pData[0]->info.type;
SAvgRes* pAvgRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
// computing based on the true data block
SColumnInfoData* pCol = pInput->pData[0];
int32_t start = pInput->startRowIndex;
int32_t numOfRows = pInput->numOfRows;
switch (type) {
case TSDB_DATA_TYPE_TINYINT: {
int8_t* plist = (int8_t*)pCol->pData;
for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
continue;
}
numOfElem += 1;
pAvgRes->count += 1;
pAvgRes->sum.isum += plist[i];
}
break;
}
case TSDB_DATA_TYPE_SMALLINT: {
int16_t* plist = (int16_t*)pCol->pData;
for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
continue;
}
numOfElem += 1;
pAvgRes->count += 1;
pAvgRes->sum.isum += plist[i];
}
break;
}
case TSDB_DATA_TYPE_INT: {
int32_t* plist = (int32_t*)pCol->pData;
for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
continue;
}
numOfElem += 1;
pAvgRes->count += 1;
pAvgRes->sum.isum += plist[i];
}
break;
}
case TSDB_DATA_TYPE_BIGINT: {
int64_t* plist = (int64_t*)pCol->pData;
for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
continue;
}
numOfElem += 1;
pAvgRes->count += 1;
pAvgRes->sum.isum += plist[i];
}
break;
}
case TSDB_DATA_TYPE_FLOAT: {
float* plist = (float*)pCol->pData;
for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
continue;
}
numOfElem += 1;
pAvgRes->count += 1;
pAvgRes->sum.dsum += plist[i];
}
break;
}
case TSDB_DATA_TYPE_DOUBLE: {
double* plist = (double*)pCol->pData;
for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
continue;
}
numOfElem += 1;
pAvgRes->count += 1;
pAvgRes->sum.dsum += plist[i];
}
break;
}
default:
break;
}
// data in the check operation are all null, not output
SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1);
return TSDB_CODE_SUCCESS;
}
int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) {
SInputColumnInfoData* pInput = &pCtx->input;
int32_t type = pInput->pData[0]->info.type;
SAvgRes* pAvgRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
if (IS_INTEGER_TYPE(type)) {
pAvgRes->result = pAvgRes->sum.isum / ((double) pAvgRes->count);
} else {
pAvgRes->result = pAvgRes->sum.dsum / ((double) pAvgRes->count);
}
return functionFinalize(pCtx, pBlock, slotId);
}
EFuncDataRequired statisDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow){
return FUNC_DATA_REQUIRED_STATIS_LOAD;
}
@ -292,49 +521,6 @@ bool getMinmaxFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
return true;
}
#define GET_TS_LIST(x) ((TSKEY*)((x)->ptsList))
#define GET_TS_DATA(x, y) (GET_TS_LIST(x)[(y)])
#define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx) \
do { \
for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \
SqlFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \
__ctx->fpSet.process(__ctx); \
} \
} while (0);
#define DO_UPDATE_SUBSID_RES(ctx, ts) \
do { \
for (int32_t _i = 0; _i < (ctx)->subsidiaryRes.numOfCols; ++_i) { \
SqlFunctionCtx *__ctx = (ctx)->subsidiaryRes.pCtx[_i]; \
if (__ctx->functionId == FUNCTION_TS_DUMMY) { \
__ctx->tag.i = (ts); \
__ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \
} \
__ctx->fpSet.process(__ctx); \
} \
} while (0)
#define UPDATE_DATA(ctx, left, right, num, sign, _ts) \
do { \
if (((left) < (right)) ^ (sign)) { \
(left) = (right); \
DO_UPDATE_SUBSID_RES(ctx, _ts); \
(num) += 1; \
} \
} while (0)
#define LOOPCHECK_N(val, _col, ctx, _t, _nrow, _start, sign, num) \
do { \
_t *d = (_t *)((_col)->pData); \
for (int32_t i = (_start); i < (_nrow) + (_start); ++i) { \
if (((_col)->hasNull) && colDataIsNull_f((_col)->nullbitmap, i)) { \
continue; \
} \
TSKEY ts = (ctx)->ptsList != NULL ? GET_TS_DATA(ctx, i) : 0; \
UPDATE_DATA(ctx, val, d[i], num, sign, ts); \
} \
} while (0)
int32_t doMinMaxHelper(SqlFunctionCtx *pCtx, int32_t isMinFunc) {
int32_t numOfElems = 0;
@ -479,13 +665,6 @@ int32_t maxFunction(SqlFunctionCtx *pCtx) {
return TSDB_CODE_SUCCESS;
}
typedef struct SStddevRes {
double result;
int64_t count;
union {double quadraticDSum; int64_t quadraticISum;};
union {double dsum; int64_t isum;};
} SStddevRes;
bool getStddevFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
pEnv->calcMemSize = sizeof(SStddevRes);
return true;
@ -588,8 +767,8 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx) {
numOfElem += 1;
pStddevRes->count += 1;
pStddevRes->isum += plist[i];
pStddevRes->quadraticISum += plist[i] * plist[i];
pStddevRes->dsum += plist[i];
pStddevRes->quadraticDSum += plist[i] * plist[i];
}
break;
}
@ -603,8 +782,8 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx) {
numOfElem += 1;
pStddevRes->count += 1;
pStddevRes->isum += plist[i];
pStddevRes->quadraticISum += plist[i] * plist[i];
pStddevRes->dsum += plist[i];
pStddevRes->quadraticDSum += plist[i] * plist[i];
}
break;
}
@ -619,21 +798,21 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx) {
}
int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) {
SInputColumnInfoData* pInput = &pCtx->input;
int32_t type = pInput->pData[0]->info.type;
SStddevRes* pStddevRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
double avg = pStddevRes->isum / ((double) pStddevRes->count);
pStddevRes->result = sqrt(pStddevRes->quadraticISum/((double)pStddevRes->count) - avg*avg);
double avg;
if (IS_INTEGER_TYPE(type)) {
avg = pStddevRes->isum / ((double) pStddevRes->count);
pStddevRes->result = sqrt(pStddevRes->quadraticISum/((double)pStddevRes->count) - avg*avg);
} else {
avg = pStddevRes->dsum / ((double) pStddevRes->count);
pStddevRes->result = sqrt(pStddevRes->quadraticDSum/((double)pStddevRes->count) - avg*avg);
}
return functionFinalize(pCtx, pBlock, slotId);
}
typedef struct SPercentileInfo {
double result;
tMemBucket *pMemBucket;
int32_t stage;
double minval;
double maxval;
int64_t numOfElems;
} SPercentileInfo;
bool getPercentileFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
pEnv->calcMemSize = sizeof(SPercentileInfo);
return true;
@ -928,14 +1107,6 @@ int32_t lastFunction(SqlFunctionCtx *pCtx) {
return TSDB_CODE_SUCCESS;
}
typedef struct SDiffInfo {
bool hasPrev;
bool includeNull;
bool ignoreNegative;
bool firstOutput;
union { int64_t i64; double d64;} prev;
} SDiffInfo;
bool getDiffFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
pEnv->calcMemSize = sizeof(SDiffInfo);
return true;
@ -1168,21 +1339,6 @@ int32_t diffFunction(SqlFunctionCtx *pCtx) {
}
}
typedef struct STopBotResItem {
SVariant v;
uint64_t uid; // it is a table uid, used to extract tag data during building of the final result for the tag data
struct {
int32_t pageId;
int32_t offset;
} tuplePos; // tuple data of this chosen row
} STopBotResItem;
typedef struct STopBotRes {
int32_t pageId;
// int32_t num;
STopBotResItem *pItems;
} STopBotRes;
bool getTopBotFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
SValueNode* pkNode = (SValueNode*) nodesListGetNode(pFunc->pParameterList, 1);
pEnv->calcMemSize = sizeof(STopBotRes) + pkNode->datum.i * sizeof(STopBotResItem);
@ -1335,4 +1491,4 @@ int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId
return pEntryInfo->numOfRes;
// return functionFinalize(pCtx, pBlock, slotId);
}
}

161
source/libs/function/src/tudf.c
View File

@ -124,7 +124,7 @@ enum {
int64_t gUdfTaskSeqNum = 0;
typedef struct SUdfdProxy {
int32_t dnodeId;
char udfdPipeName[UDF_LISTEN_PIPE_NAME_LEN];
uv_barrier_t gUdfInitBarrier;
uv_loop_t gUdfdLoop;
@ -137,11 +137,11 @@ typedef struct SUdfdProxy {
int8_t gUdfcState;
QUEUE gUdfTaskQueue;
QUEUE gUvProcTaskQueue;
// int8_t gUdfcState = UDFC_STATE_INITAL;
// QUEUE gUdfTaskQueue = {0};
// QUEUE gUvProcTaskQueue = {0};
int8_t initialized;
} SUdfdProxy;
SUdfdProxy gUdfdProxy = {0};
typedef struct SUdfUvSession {
SUdfdProxy *udfc;
@ -209,19 +209,27 @@ enum {
UDFC_STATE_STARTNG, // starting after udfcOpen
UDFC_STATE_READY, // started and begin to receive quests
UDFC_STATE_STOPPING, // stopping after udfcClose
UDFC_STATUS_FINAL, // stopped
};
int32_t getUdfdPipeName(char* pipeName, int32_t size) {
char dnodeId[8] = {0};
size_t dnodeIdSize;
int32_t err = uv_os_getenv(UDF_DNODE_ID_ENV_NAME, dnodeId, &dnodeIdSize);
if (err != 0) {
dnodeId[0] = '1';
}
snprintf(pipeName, size, "%s%s", UDF_LISTEN_PIPE_NAME_PREFIX, dnodeId);
return 0;
}
int32_t encodeUdfSetupRequest(void **buf, const SUdfSetupRequest *setup) {
int32_t len = 0;
len += taosEncodeBinary(buf, setup->udfName, TSDB_FUNC_NAME_LEN);
len += taosEncodeSEpSet(buf, &setup->epSet);
return len;
}
void* decodeUdfSetupRequest(const void* buf, SUdfSetupRequest *request) {
buf = taosDecodeBinaryTo(buf, request->udfName, TSDB_FUNC_NAME_LEN);
buf = taosDecodeSEpSet((void*)buf, &request->epSet);
return (void*)buf;
}
@ -604,7 +612,7 @@ void onUdfcPipeClose(uv_handle_t *handle) {
}
int32_t udfcGetUvTaskResponseResult(SClientUdfTask *task, SClientUvTaskNode *uvTask) {
debugPrint("%s", "get uv task result");
fnDebug("udfc get uv task result. task: %p", task);
if (uvTask->type == UV_TASK_REQ_RSP) {
if (uvTask->rspBuf.base != NULL) {
SUdfResponse rsp;
@ -647,7 +655,6 @@ int32_t udfcGetUvTaskResponseResult(SClientUdfTask *task, SClientUvTaskNode *uvT
}
void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) {
debugPrint("%s", "client allocate buffer to receive from pipe");
SClientUvConn *conn = handle->data;
SClientConnBuf *connBuf = &conn->readBuf;
@ -662,7 +669,7 @@ void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf
buf->base = connBuf->buf;
buf->len = connBuf->cap;
} else {
//TODO: log error
fnError("udfc allocate buffer failure. size: %d", msgHeadSize);
buf->base = NULL;
buf->len = 0;
}
@ -674,13 +681,13 @@ void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf
buf->base = connBuf->buf + connBuf->len;
buf->len = connBuf->cap - connBuf->len;
} else {
//TODO: log error free connBuf->buf
fnError("udfc re-allocate buffer failure. size: %d", connBuf->cap);
buf->base = NULL;
buf->len = 0;
}
}
debugPrint("\tconn buf cap - len - total : %d - %d - %d", connBuf->cap, connBuf->len, connBuf->total);
fnTrace("conn buf cap - len - total : %d - %d - %d", connBuf->cap, connBuf->len, connBuf->total);
}
@ -689,6 +696,7 @@ bool isUdfcUvMsgComplete(SClientConnBuf *connBuf) {
connBuf->total = *(int32_t *) (connBuf->buf);
}
if (connBuf->len == connBuf->cap && connBuf->total == connBuf->cap) {
fnTrace("udfc complete message is received, now handle it");
return true;
}
return false;
@ -696,10 +704,10 @@ bool isUdfcUvMsgComplete(SClientConnBuf *connBuf) {
void udfcUvHandleRsp(SClientUvConn *conn) {
SClientConnBuf *connBuf = &conn->readBuf;
int64_t seqNum = *(int64_t *) (connBuf->buf + sizeof(int32_t)); // msglen int32_t then seqnum
int64_t seqNum = *(int64_t *) (connBuf->buf + sizeof(int32_t)); // msglen then seqnum
if (QUEUE_EMPTY(&conn->taskQueue)) {
//LOG error
fnError("udfc no task waiting for response on connection");
return;
}
bool found = false;
@ -713,7 +721,7 @@ void udfcUvHandleRsp(SClientUvConn *conn) {
found = true;
taskFound = task;
} else {
//LOG error;
fnError("udfc more than one task waiting for the same response");
continue;
}
}
@ -727,7 +735,7 @@ void udfcUvHandleRsp(SClientUvConn *conn) {
uv_sem_post(&taskFound->taskSem);
QUEUE_REMOVE(&taskFound->procTaskQueue);
} else {
//TODO: LOG error
fnError("no task is waiting for the response.");
}
connBuf->buf = NULL;
connBuf->total = -1;
@ -751,7 +759,7 @@ void udfcUvHandleError(SClientUvConn *conn) {
}
void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) {
debugPrint("%s, nread: %zd", "client read from pipe", nread);
fnTrace("udfc client %p, client read from pipe. nread: %zd", client, nread);
if (nread == 0) return;
SClientUvConn *conn = client->data;
@ -764,9 +772,9 @@ void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) {
}
if (nread < 0) {
debugPrint("\tclient read error: %s", uv_strerror(nread));
fnError("udfc client pipe %p read error: %s", client, uv_strerror(nread));
if (nread == UV_EOF) {
//TODO:
fnError("udfc client pipe %p closed", client);
}
udfcUvHandleError(conn);
}
@ -774,16 +782,15 @@ void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) {
}
void onUdfClientWrite(uv_write_t *write, int status) {
debugPrint("%s", "after writing to pipe");
SClientUvTaskNode *uvTask = write->data;
uv_pipe_t *pipe = uvTask->pipe;
if (status == 0) {
uv_pipe_t *pipe = uvTask->pipe;
SClientUvConn *conn = pipe->data;
QUEUE_INSERT_TAIL(&conn->taskQueue, &uvTask->connTaskQueue);
} else {
//TODO Log error;
fnError("udfc client %p write error.", pipe);
}
debugPrint("\tlength:%zu", uvTask->reqBuf.len);
fnTrace("udfc client %p write length:%zu", pipe, uvTask->reqBuf.len);
taosMemoryFree(write);
taosMemoryFree(uvTask->reqBuf.base);
}
@ -841,7 +848,7 @@ int32_t createUdfcUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskN
}
int32_t queueUvUdfTask(SClientUvTaskNode *uvTask) {
debugPrint("%s, %d", "queue uv task", uvTask->type);
fnTrace("queue uv task to event loop, task: %d, %p", uvTask->type, uvTask);
SUdfdProxy *udfc = uvTask->udfc;
uv_mutex_lock(&udfc->gUdfTaskQueueMutex);
QUEUE_INSERT_TAIL(&udfc->gUdfTaskQueue, &uvTask->recvTaskQueue);
@ -855,7 +862,7 @@ int32_t queueUvUdfTask(SClientUvTaskNode *uvTask) {
}
int32_t startUvUdfTask(SClientUvTaskNode *uvTask) {
debugPrint("%s, type %d", "start uv task ", uvTask->type);
fnTrace("event loop start uv task. task: %d, %p", uvTask->type, uvTask);
switch (uvTask->type) {
case UV_TASK_CONNECT: {
uv_pipe_t *pipe = taosMemoryMalloc(sizeof(uv_pipe_t));
@ -874,8 +881,7 @@ int32_t startUvUdfTask(SClientUvTaskNode *uvTask) {
uv_connect_t *connReq = taosMemoryMalloc(sizeof(uv_connect_t));
connReq->data = uvTask;
uv_pipe_connect(connReq, pipe, "udf.sock", onUdfClientConnect);
uv_pipe_connect(connReq, pipe, uvTask->udfc->udfdPipeName, onUdfClientConnect);
break;
}
case UV_TASK_REQ_RSP: {
@ -971,27 +977,37 @@ void constructUdfService(void *argsThread) {
uv_loop_close(&udfc->gUdfdLoop);
}
int32_t udfcOpen(int32_t dnodeId, UdfcHandle *udfc) {
SUdfdProxy *proxy = taosMemoryCalloc(1, sizeof(SUdfdProxy));
proxy->dnodeId = dnodeId;
int32_t udfcOpen() {
int8_t old = atomic_val_compare_exchange_8(&gUdfdProxy.initialized, 0, 1);
if (old == 1) {
return 0;
}
SUdfdProxy *proxy = &gUdfdProxy;
getUdfdPipeName(proxy->udfdPipeName, UDF_LISTEN_PIPE_NAME_LEN);
proxy->gUdfcState = UDFC_STATE_STARTNG;
uv_barrier_init(&proxy->gUdfInitBarrier, 2);
uv_thread_create(&proxy->gUdfLoopThread, constructUdfService, proxy);
uv_barrier_wait(&proxy->gUdfInitBarrier);
atomic_store_8(&proxy->gUdfcState, UDFC_STATE_READY);
proxy->gUdfcState = UDFC_STATE_READY;
*udfc = proxy;
uv_barrier_wait(&proxy->gUdfInitBarrier);
fnInfo("udfc initialized")
return 0;
}
int32_t udfcClose(UdfcHandle udfcHandle) {
SUdfdProxy *udfc = udfcHandle;
int32_t udfcClose() {
int8_t old = atomic_val_compare_exchange_8(&gUdfdProxy.initialized, 1, 0);
if (old == 0) {
return 0;
}
SUdfdProxy *udfc = &gUdfdProxy;
udfc->gUdfcState = UDFC_STATE_STOPPING;
uv_async_send(&udfc->gUdfLoopStopAsync);
uv_thread_join(&udfc->gUdfLoopThread);
uv_mutex_destroy(&udfc->gUdfTaskQueueMutex);
uv_barrier_destroy(&udfc->gUdfInitBarrier);
udfc->gUdfcState = UDFC_STATUS_FINAL;
taosMemoryFree(udfc);
udfc->gUdfcState = UDFC_STATE_INITAL;
fnInfo("udfc cleaned up");
return 0;
}
@ -1009,12 +1025,15 @@ int32_t udfcRunUvTask(SClientUdfTask *task, int8_t uvTaskType) {
return task->errCode;
}
int32_t setupUdf(UdfcHandle udfc, char udfName[], SEpSet *epSet, UdfcFuncHandle *funcHandle) {
debugPrint("%s", "client setup udf");
int32_t setupUdf(char udfName[], UdfcFuncHandle *funcHandle) {
fnInfo("udfc setup udf. udfName: %s", udfName);
if (gUdfdProxy.gUdfcState != UDFC_STATE_READY) {
return UDFC_CODE_INVALID_STATE;
}
SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask));
task->errCode = 0;
task->session = taosMemoryMalloc(sizeof(SUdfUvSession));
task->session->udfc = udfc;
task->session->udfc = &gUdfdProxy;
task->type = UDF_TASK_SETUP;
SUdfSetupRequest *req = &task->_setup.req;
@ -1022,15 +1041,20 @@ int32_t setupUdf(UdfcHandle udfc, char udfName[], SEpSet *epSet, UdfcFuncHandle
int32_t errCode = udfcRunUvTask(task, UV_TASK_CONNECT);
if (errCode != 0) {
//TODO: log error
return -1;
fnError("failed to connect to pipe. udfName: %s, pipe: %s", udfName, (&gUdfdProxy)->udfdPipeName);
return UDFC_CODE_CONNECT_PIPE_ERR;
}
udfcRunUvTask(task, UV_TASK_REQ_RSP);
SUdfSetupResponse *rsp = &task->_setup.rsp;
task->session->severHandle = rsp->udfHandle;
*funcHandle = task->session;
if (task->errCode != 0) {
fnError("failed to setup udf. err: %d", task->errCode)
} else {
fnInfo("sucessfully setup udf func handle. handle: %p", task->session);
*funcHandle = task->session;
}
int32_t err = task->errCode;
taosMemoryFree(task);
return err;
@ -1038,7 +1062,7 @@ int32_t setupUdf(UdfcHandle udfc, char udfName[], SEpSet *epSet, UdfcFuncHandle
int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdfInterBuf *state, SUdfInterBuf *state2,
SSDataBlock* output, SUdfInterBuf *newState) {
debugPrint("%s", "client call udf");
fnTrace("udfc call udf. callType: %d, funcHandle: %p", callType, handle);
SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask));
task->errCode = 0;
@ -1076,35 +1100,37 @@ int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdf
udfcRunUvTask(task, UV_TASK_REQ_RSP);
SUdfCallResponse *rsp = &task->_call.rsp;
switch (callType) {
case TSDB_UDF_CALL_AGG_INIT: {
*newState = rsp->resultBuf;
break;
}
case TSDB_UDF_CALL_AGG_PROC: {
*newState = rsp->resultBuf;
break;
}
case TSDB_UDF_CALL_AGG_MERGE: {
*newState = rsp->resultBuf;
break;
}
case TSDB_UDF_CALL_AGG_FIN: {
*newState = rsp->resultBuf;
break;
}
case TSDB_UDF_CALL_SCALA_PROC: {
*output = rsp->resultData;
break;
if (task->errCode != 0) {
fnError("call udf failure. err: %d", task->errCode);
} else {
SUdfCallResponse *rsp = &task->_call.rsp;
switch (callType) {
case TSDB_UDF_CALL_AGG_INIT: {
*newState = rsp->resultBuf;
break;
}
case TSDB_UDF_CALL_AGG_PROC: {
*newState = rsp->resultBuf;
break;
}
case TSDB_UDF_CALL_AGG_MERGE: {
*newState = rsp->resultBuf;
break;
}
case TSDB_UDF_CALL_AGG_FIN: {
*newState = rsp->resultBuf;
break;
}
case TSDB_UDF_CALL_SCALA_PROC: {
*output = rsp->resultData;
break;
}
}
}
taosMemoryFree(task);
return task->errCode;
}
//TODO: translate these calls to callUdf
int32_t callUdfAggInit(UdfcFuncHandle handle, SUdfInterBuf *interBuf) {
int8_t callType = TSDB_UDF_CALL_AGG_INIT;
@ -1148,7 +1174,7 @@ int32_t callUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t nu
}
int32_t teardownUdf(UdfcFuncHandle handle) {
debugPrint("%s", "client teardown udf");
fnInfo("tear down udf. udf func handle: %p", handle);
SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask));
task->errCode = 0;
@ -1160,7 +1186,6 @@ int32_t teardownUdf(UdfcFuncHandle handle) {
udfcRunUvTask(task, UV_TASK_REQ_RSP);
SUdfTeardownResponse *rsp = &task->_teardown.rsp;
int32_t err = task->errCode;

103
source/libs/function/src/udfd.c
View File

@ -20,6 +20,7 @@
#include "tudf.h"
#include "tudfInt.h"
#include "tdatablock.h"
#include "tdataformat.h"
#include "tglobal.h"
#include "tmsg.h"
@ -31,8 +32,9 @@ typedef struct SUdfdContext {
uv_signal_t intrSignal;
char listenPipeName[UDF_LISTEN_PIPE_NAME_LEN];
uv_pipe_t listeningPipe;
void *clientRpc;
void *clientRpc;
SCorEpSet mgmtEp;
uv_mutex_t udfsMutex;
SHashObj *udfsHash;
@ -63,8 +65,13 @@ typedef struct SUdf {
uv_mutex_t lock;
uv_cond_t condReady;
char name[16];
int8_t type;
char name[TSDB_FUNC_NAME_LEN];
int8_t funcType;
int8_t scriptType;
int8_t outputType;
int32_t outputLen;
int32_t bufSize;
char path[PATH_MAX];
uv_lib_t lib;
@ -78,17 +85,17 @@ typedef struct SUdfcFuncHandle {
SUdf *udf;
} SUdfcFuncHandle;
int32_t udfdFillUdfInfoFromMNode(void *clientRpc, SEpSet *pEpSet, char *udfName, SUdf *udf);
int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf);
int32_t udfdLoadUdf(char *udfName, SEpSet *pEpSet, SUdf *udf) {
int32_t udfdLoadUdf(char *udfName, SUdf *udf) {
strcpy(udf->name, udfName);
udfdFillUdfInfoFromMNode(global.clientRpc, pEpSet, udf->name, udf);
udfdFillUdfInfoFromMNode(global.clientRpc, udf->name, udf);
//strcpy(udf->path, "/home/slzhou/TDengine/debug/build/lib/libudf1.so");
int err = uv_dlopen(udf->path, &udf->lib);
if (err != 0) {
fnError("can not load library %s. error: %s", udf->path, uv_strerror(err));
// TODO set error
return UDFC_CODE_LOAD_UDF_FAILURE;
}
// TODO: find all the functions
char normalFuncName[TSDB_FUNC_NAME_LEN] = {0};
@ -115,8 +122,8 @@ void udfdProcessRequest(uv_work_t *req) {
SUdf *udf = NULL;
uv_mutex_lock(&global.udfsMutex);
SUdf **udfInHash = taosHashGet(global.udfsHash, request.setup.udfName, TSDB_FUNC_NAME_LEN);
if (*udfInHash) {
SUdf **udfInHash = taosHashGet(global.udfsHash, request.setup.udfName, strlen(request.setup.udfName));
if (udfInHash) {
++(*udfInHash)->refCount;
udf = *udfInHash;
uv_mutex_unlock(&global.udfsMutex);
@ -128,14 +135,14 @@ void udfdProcessRequest(uv_work_t *req) {
uv_mutex_init(&udfNew->lock);
uv_cond_init(&udfNew->condReady);
udf = udfNew;
taosHashPut(global.udfsHash, request.setup.udfName, TSDB_FUNC_NAME_LEN, &udfNew, sizeof(&udfNew));
taosHashPut(global.udfsHash, request.setup.udfName, strlen(request.setup.udfName), &udfNew, sizeof(&udfNew));
uv_mutex_unlock(&global.udfsMutex);
}
uv_mutex_lock(&udf->lock);
if (udf->state == UDF_STATE_INIT) {
udf->state = UDF_STATE_LOADING;
udfdLoadUdf(setup->udfName, &setup->epSet, udf);
udfdLoadUdf(setup->udfName, udf);
udf->state = UDF_STATE_READY;
uv_cond_broadcast(&udf->condReady);
uv_mutex_unlock(&udf->lock);
@ -214,7 +221,7 @@ void udfdProcessRequest(uv_work_t *req) {
udf->refCount--;
if (udf->refCount == 0) {
unloadUdf = true;
taosHashRemove(global.udfsHash, udf->name, TSDB_FUNC_NAME_LEN);
taosHashRemove(global.udfsHash, udf->name, strlen(udf->name));
}
uv_mutex_unlock(&global.udfsMutex);
if (unloadUdf) {
@ -393,7 +400,48 @@ void udfdIntrSignalHandler(uv_signal_t *handle, int signum) {
void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { return; }
int32_t udfdFillUdfInfoFromMNode(void *clientRpc, SEpSet *pEpSet, char *udfName, SUdf *udf) {
int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet) {
pEpSet->version = 0;
// init mnode ip set
SEpSet* mgmtEpSet = &(pEpSet->epSet);
mgmtEpSet->numOfEps = 0;
mgmtEpSet->inUse = 0;
if (firstEp && firstEp[0] != 0) {
if (strlen(firstEp) >= TSDB_EP_LEN) {
terrno = TSDB_CODE_TSC_INVALID_FQDN;
return -1;
}
int32_t code = taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[0]);
if (code != TSDB_CODE_SUCCESS) {
terrno = TSDB_CODE_TSC_INVALID_FQDN;
return terrno;
}
mgmtEpSet->numOfEps++;
}
if (secondEp && secondEp[0] != 0) {
if (strlen(secondEp) >= TSDB_EP_LEN) {
terrno = TSDB_CODE_TSC_INVALID_FQDN;
return -1;
}
taosGetFqdnPortFromEp(secondEp, &mgmtEpSet->eps[mgmtEpSet->numOfEps]);
mgmtEpSet->numOfEps++;
}
if (mgmtEpSet->numOfEps == 0) {
terrno = TSDB_CODE_TSC_INVALID_FQDN;
return -1;
}
return 0;
}
int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf) {
SRetrieveFuncReq retrieveReq = {0};
retrieveReq.numOfFuncs = 1;
retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN);
@ -410,15 +458,21 @@ int32_t udfdFillUdfInfoFromMNode(void *clientRpc, SEpSet *pEpSet, char *udfName,
rpcMsg.msgType = TDMT_MND_RETRIEVE_FUNC;
SRpcMsg rpcRsp = {0};
rpcSendRecv(clientRpc, pEpSet, &rpcMsg, &rpcRsp);
rpcSendRecv(clientRpc, &global.mgmtEp.epSet, &rpcMsg, &rpcRsp);
SRetrieveFuncRsp retrieveRsp = {0};
tDeserializeSRetrieveFuncRsp(rpcRsp.pCont, rpcRsp.contLen, &retrieveRsp);
SFuncInfo *pFuncInfo = (SFuncInfo *)taosArrayGet(retrieveRsp.pFuncInfos, 0);
udf->funcType = pFuncInfo->funcType;
udf->scriptType = pFuncInfo->scriptType;
udf->outputType = pFuncInfo->funcType;
udf->outputLen = pFuncInfo->outputLen;
udf->bufSize = pFuncInfo->bufSize;
char path[PATH_MAX] = {0};
taosGetTmpfilePath("/tmp", "libudf", path);
TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);
snprintf(path, sizeof(path), "%s/lib%s.so", "/tmp", udfName);
TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL);
// TODO check for failure of flush to disk
taosWriteFile(file, pFuncInfo->pCode, pFuncInfo->codeSize);
taosCloseFile(&file);
@ -531,15 +585,7 @@ static int32_t udfdUvInit() {
uv_pipe_open(&global.ctrlPipe, 0);
uv_read_start((uv_stream_t *)&global.ctrlPipe, udfdCtrlAllocBufCb, udfdCtrlReadCb);
char dnodeId[8] = {0};
size_t dnodeIdSize;
int32_t err = uv_os_getenv("DNODE_ID", dnodeId, &dnodeIdSize);
if (err != 0) {
dnodeId[0] = '1';
}
char listenPipeName[32] = {0};
snprintf(listenPipeName, sizeof(listenPipeName), "%s%s", UDF_LISTEN_PIPE_NAME_PREFIX, dnodeId);
strcpy(global.listenPipeName, listenPipeName);
getUdfdPipeName(global.listenPipeName, UDF_LISTEN_PIPE_NAME_LEN);
removeListeningPipe();
@ -550,7 +596,7 @@ static int32_t udfdUvInit() {
int r;
fnInfo("bind to pipe %s", global.listenPipeName);
if ((r = uv_pipe_bind(&global.listeningPipe, listenPipeName))) {
if ((r = uv_pipe_bind(&global.listeningPipe, global.listenPipeName))) {
fnError("Bind error %s", uv_err_name(r));
removeListeningPipe();
return -1;
@ -580,7 +626,7 @@ static int32_t udfdRun() {
fnInfo("start the udfd");
int code = uv_run(global.loop, UV_RUN_DEFAULT);
fnInfo("udfd stopped. result: %s", uv_err_name(code));
fnInfo("udfd stopped. result: %s, code: %d", uv_err_name(code), code);
int codeClose = uv_loop_close(global.loop);
fnDebug("uv loop close. result: %s", uv_err_name(codeClose));
udfdCloseClientRpc();
@ -615,5 +661,6 @@ int main(int argc, char *argv[]) {
return -1;
}
initEpSetFromCfg(tsFirst, tsSecond, &global.mgmtEp);
return udfdRun();
}

121
source/libs/function/test/runUdf.c
View File

@ -1,61 +1,84 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "uv.h"
#include "fnLog.h"
#include "os.h"
#include "tudf.h"
#include "tdatablock.h"
#include "tglobal.h"
#include "tudf.h"
static int32_t parseArgs(int32_t argc, char *argv[]) {
for (int32_t i = 1; i < argc; ++i) {
if (strcmp(argv[i], "-c") == 0) {
if (i < argc - 1) {
if (strlen(argv[++i]) >= PATH_MAX) {
printf("config file path overflow");
return -1;
}
tstrncpy(configDir, argv[i], PATH_MAX);
} else {
printf("'-c' requires a parameter, default is %s\n", configDir);
return -1;
}
}
}
return 0;
}
static int32_t initLog() {
char logName[12] = {0};
snprintf(logName, sizeof(logName), "%slog", "udfc");
return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, 0);
}
int main(int argc, char *argv[]) {
UdfcHandle udfc;
udfcOpen(1, &udfc);
uv_sleep(1000);
char path[256] = {0};
size_t cwdSize = 256;
int err = uv_cwd(path, &cwdSize);
if (err != 0) {
fprintf(stderr, "err cwd: %s\n", uv_strerror(err));
return err;
parseArgs(argc, argv);
initLog();
if (taosInitCfg(configDir, NULL, NULL, NULL, 0) != 0) {
fnError("failed to start since read config error");
return -1;
}
udfcOpen();
uv_sleep(1000);
UdfcFuncHandle handle;
setupUdf("udf1", &handle);
SSDataBlock block = {0};
SSDataBlock *pBlock = &block;
pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
pBlock->info.numOfCols = 1;
pBlock->info.rows = 4;
char data[16] = {0};
char bitmap[4] = {0};
for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
SColumnInfoData colInfo = {0};
colInfo.info.type = TSDB_DATA_TYPE_INT;
colInfo.info.bytes = sizeof(int32_t);
colInfo.info.colId = 1;
colInfo.pData = data;
colInfo.nullbitmap = bitmap;
for (int32_t j = 0; j < pBlock->info.rows; ++j) {
colDataAppendInt32(&colInfo, j, &j);
}
fprintf(stdout, "current working directory:%s\n", path);
strcat(path, "/libudf1.so");
taosArrayPush(pBlock->pDataBlock, &colInfo);
}
UdfcFuncHandle handle;
SEpSet epSet;
setupUdf(udfc, "udf1", &epSet, &handle);
SScalarParam input = {0};
input.numOfRows = pBlock->info.rows;
input.columnData = taosArrayGet(pBlock->pDataBlock, 0);
SScalarParam output = {0};
callUdfScalarFunc(handle, &input, 1, &output);
SSDataBlock block = {0};
SSDataBlock* pBlock = &block;
pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
pBlock->info.numOfCols = 1;
pBlock->info.rows = 4;
char data[16] = {0};
char bitmap[4] = {0};
for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
SColumnInfoData colInfo = {0};
colInfo.info.type = TSDB_DATA_TYPE_INT;
colInfo.info.bytes = sizeof(int32_t);
colInfo.info.colId = 1;
colInfo.pData = data;
colInfo.nullbitmap = bitmap;
for (int32_t j = 0; j < pBlock->info.rows; ++j) {
colDataAppendInt32(&colInfo, j, &j);
}
taosArrayPush(pBlock->pDataBlock, &colInfo);
}
SScalarParam input = {0};
input.numOfRows = pBlock->info.rows;
input.columnData = taosArrayGet(pBlock->pDataBlock, 0);
SScalarParam output = {0};
callUdfScalarFunc(handle, &input, 1 , &output);
SColumnInfoData *col = output.columnData;
for (int32_t i = 0; i < output.numOfRows; ++i) {
fprintf(stderr, "%d\t%d\n" , i, *(int32_t*)(col->pData + i *sizeof(int32_t)));
}
teardownUdf(handle);
udfcClose(udfc);
SColumnInfoData *col = output.columnData;
for (int32_t i = 0; i < output.numOfRows; ++i) {
fprintf(stderr, "%d\t%d\n", i, *(int32_t *)(col->pData + i * sizeof(int32_t)));
}
teardownUdf(handle);
udfcClose();
}

34
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -671,9 +671,6 @@ static int32_t jsonToName(const SJson* pJson, void* pObj) {
static const char* jkScanPhysiPlanScanCols = "ScanCols";
static const char* jkScanPhysiPlanTableId = "TableId";
static const char* jkScanPhysiPlanTableType = "TableType";
static const char* jkScanPhysiPlanScanOrder = "ScanOrder";
static const char* jkScanPhysiPlanScanCount = "ScanCount";
static const char* jkScanPhysiPlanReverseScanCount = "ReverseScanCount";
static const char* jkScanPhysiPlanTableName = "TableName";
static int32_t physiScanNodeToJson(const void* pObj, SJson* pJson) {
@ -689,15 +686,6 @@ static int32_t physiScanNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanTableType, pNode->tableType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanScanOrder, pNode->order);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanScanCount, pNode->count);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanReverseScanCount, pNode->reverse);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkScanPhysiPlanTableName, nameToJson, &pNode->tableName);
}
@ -718,15 +706,6 @@ static int32_t jsonToPhysiScanNode(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetTinyIntValue(pJson, jkScanPhysiPlanTableType, &pNode->tableType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetIntValue(pJson, jkScanPhysiPlanScanOrder, &pNode->order);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetIntValue(pJson, jkScanPhysiPlanScanCount, &pNode->count);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetIntValue(pJson, jkScanPhysiPlanReverseScanCount, &pNode->reverse);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonToObject(pJson, jkScanPhysiPlanTableName, jsonToName, &pNode->tableName);
}
@ -742,7 +721,8 @@ static int32_t jsonToPhysiTagScanNode(const SJson* pJson, void* pObj) {
return jsonToPhysiScanNode(pJson, pObj);
}
static const char* jkTableScanPhysiPlanScanFlag = "ScanFlag";
static const char* jkTableScanPhysiPlanScanCount = "ScanCount";
static const char* jkTableScanPhysiPlanReverseScanCount = "ReverseScanCount";
static const char* jkTableScanPhysiPlanStartKey = "StartKey";
static const char* jkTableScanPhysiPlanEndKey = "EndKey";
static const char* jkTableScanPhysiPlanRatio = "Ratio";
@ -759,7 +739,10 @@ static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) {
int32_t code = physiScanNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanScanFlag, pNode->scanFlag);
code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanScanCount, pNode->scanSeq[0]);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanReverseScanCount, pNode->scanSeq[1]);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanStartKey, pNode->scanRange.skey);
@ -800,7 +783,10 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) {
int32_t code = jsonToPhysiScanNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetUTinyIntValue(pJson, jkTableScanPhysiPlanScanFlag, &pNode->scanFlag);
code = tjsonGetUTinyIntValue(pJson, jkTableScanPhysiPlanScanCount, &pNode->scanSeq[0]);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetUTinyIntValue(pJson, jkTableScanPhysiPlanReverseScanCount, &pNode->scanSeq[1]);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBigIntValue(pJson, jkTableScanPhysiPlanStartKey, &pNode->scanRange.skey);

1
source/libs/parser/inc/sql.y
View File

@ -611,6 +611,7 @@ function_expression(A) ::= function_name(B) NK_LP expression_list(C) NK_RP(D).
function_expression(A) ::= star_func(B) NK_LP star_func_para_list(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); }
function_expression(A) ::= CAST(B) NK_LP expression(C) AS type_name(D) NK_RP(E). { A = createRawExprNodeExt(pCxt, &B, &E, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, C), D)); }
function_expression(A) ::= noarg_func(B) NK_LP NK_RP(C). { A = createRawExprNodeExt(pCxt, &B, &C, createFunctionNodeNoArg(pCxt, &B)); }
//function_expression(A) ::= NOW(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
%type noarg_func { SToken }
%destructor noarg_func { }

2
source/libs/parser/src/parCalcConst.c
View File

@ -83,7 +83,7 @@ static EDealRes calcConstOperator(SOperatorNode** pNode, void* pContext) {
static EDealRes calcConstFunction(SFunctionNode** pNode, void* pContext) {
SFunctionNode* pFunc = *pNode;
if (!fmIsScalarFunc(pFunc->funcId)) {
if (!fmIsScalarFunc(pFunc->funcId) || fmIsUserDefinedFunc(pFunc->funcId)) {
return DEAL_RES_CONTINUE;
}
SNode* pParam = NULL;

81
source/libs/parser/src/parTranslater.c
View File

@ -543,7 +543,9 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) {
TSDB_DATA_TYPE_BLOB == rdt.type) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
}
if (TSDB_DATA_TYPE_TIMESTAMP == ldt.type && TSDB_DATA_TYPE_TIMESTAMP == rdt.type) {
if ((TSDB_DATA_TYPE_TIMESTAMP == ldt.type && TSDB_DATA_TYPE_TIMESTAMP == rdt.type) ||
(TSDB_DATA_TYPE_TIMESTAMP == ldt.type && IS_VAR_DATA_TYPE(rdt.type)) ||
(TSDB_DATA_TYPE_TIMESTAMP == rdt.type && IS_VAR_DATA_TYPE(ldt.type))) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
}
@ -2614,37 +2616,62 @@ static int32_t translateDropComponentNode(STranslateContext* pCxt, SDropComponen
(FSerializeFunc)tSerializeSCreateDropMQSBNodeReq, &dropReq);
}
static int32_t translateCreateTopic(STranslateContext* pCxt, SCreateTopicStmt* pStmt) {
SCMCreateTopicReq createReq = {0};
static int32_t buildCreateTopicReq(STranslateContext* pCxt, SCreateTopicStmt* pStmt, SCMCreateTopicReq* pReq) {
SName name;
// tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->topicName, strlen(pStmt->topicName));
// tNameGetFullDbName(&name, pReq->name);
tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pCxt->pParseCxt->db, pStmt->topicName, &name), pReq->name);
pReq->igExists = pStmt->ignoreExists;
pReq->withTbName = pStmt->pOptions->withTable;
pReq->withSchema = pStmt->pOptions->withSchema;
pReq->withTag = pStmt->pOptions->withTag;
if (NULL != pStmt->pQuery) {
pCxt->pParseCxt->topicQuery = true;
int32_t code = translateQuery(pCxt, pStmt->pQuery);
if (TSDB_CODE_SUCCESS == code) {
code = nodesNodeToString(pStmt->pQuery, false, &createReq.ast, NULL);
}
if (TSDB_CODE_SUCCESS != code) {
return code;
}
} else {
strcpy(createReq.subscribeDbName, pStmt->subscribeDbName);
}
createReq.sql = strdup(pCxt->pParseCxt->pSql);
if (NULL == createReq.sql) {
pReq->sql = strdup(pCxt->pParseCxt->pSql);
if (NULL == pReq->sql) {
return TSDB_CODE_OUT_OF_MEMORY;
}
SName name;
// tNameSetDbName(&name, pCxt->pParseCxt->acctId, pCxt->pParseCxt->db, strlen(pCxt->pParseCxt->db));
// tNameGetFullDbName(&name, createReq.name);
tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pCxt->pParseCxt->db, pStmt->topicName, &name), createReq.name);
createReq.igExists = pStmt->ignoreExists;
createReq.withTbName = pStmt->pOptions->withTable;
createReq.withSchema = pStmt->pOptions->withSchema;
createReq.withTag = pStmt->pOptions->withTag;
int32_t code = TSDB_CODE_SUCCESS;
int32_t code = buildCmdMsg(pCxt, TDMT_MND_CREATE_TOPIC, (FSerializeFunc)tSerializeSCMCreateTopicReq, &createReq);
if (NULL != pStmt->pQuery) {
strcpy(pReq->subscribeDbName, ((SRealTableNode*)(((SSelectStmt*)pStmt->pQuery)->pFromTable))->table.dbName);
pCxt->pParseCxt->topicQuery = true;
code = translateQuery(pCxt, pStmt->pQuery);
if (TSDB_CODE_SUCCESS == code) {
code = nodesNodeToString(pStmt->pQuery, false, &pReq->ast, NULL);
}
} else {
strcpy(pReq->subscribeDbName, pStmt->subscribeDbName);
}
return code;
}
static int32_t checkCreateTopic(STranslateContext* pCxt, SCreateTopicStmt* pStmt) {
if (NULL == pStmt->pQuery) {
return TSDB_CODE_SUCCESS;
}
if (QUERY_NODE_SELECT_STMT == nodeType(pStmt->pQuery)) {
SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery;
if (!pSelect->isDistinct && QUERY_NODE_REAL_TABLE == nodeType(pSelect->pFromTable) && NULL == pSelect->pGroupByList &&
NULL == pSelect->pLimit && NULL == pSelect->pSlimit && NULL == pSelect->pOrderByList && NULL == pSelect->pPartitionByList) {
return TSDB_CODE_SUCCESS;
}
}
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TOPIC_QUERY);
}
static int32_t translateCreateTopic(STranslateContext* pCxt, SCreateTopicStmt* pStmt) {
SCMCreateTopicReq createReq = {0};
int32_t code = checkCreateTopic(pCxt, pStmt);
if (TSDB_CODE_SUCCESS == code) {
code = buildCreateTopicReq(pCxt, pStmt, &createReq);
}
if (TSDB_CODE_SUCCESS == code) {
code = buildCmdMsg(pCxt, TDMT_MND_CREATE_TOPIC, (FSerializeFunc)tSerializeSCMCreateTopicReq, &createReq);
}
tFreeSCMCreateTopicReq(&createReq);
return code;
}

2
source/libs/parser/src/parUtil.c
View File

@ -128,6 +128,8 @@ static char* getSyntaxErrFormat(int32_t errCode) {
return "soffset/offset can not be less than 0";
case TSDB_CODE_PAR_SLIMIT_LEAK_PARTITION_BY:
return "slimit/soffset only available for PARTITION BY query";
case TSDB_CODE_PAR_INVALID_TOPIC_QUERY:
return "Invalid topic query";
case TSDB_CODE_OUT_OF_MEMORY:
return "Out of memory";
default:

3
source/libs/planner/src/planLogicCreater.c
View File

@ -199,7 +199,8 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
TSWAP(pScan->pMeta, pRealTable->pMeta, STableMeta*);
TSWAP(pScan->pVgroupList, pRealTable->pVgroupList, SVgroupsInfo*);
pScan->scanFlag = MAIN_SCAN;
pScan->scanSeq[0] = 1;
pScan->scanSeq[1] = 0;
pScan->scanRange = TSWINDOW_INITIALIZER;
pScan->tableName.type = TSDB_TABLE_NAME_T;
pScan->tableName.acctId = pCxt->pPlanCxt->acctId;

251
source/libs/planner/src/planOptimizer.c
View File

@ -20,11 +20,14 @@
#define OPTIMIZE_FLAG_MASK(n) (1 << n)
#define OPTIMIZE_FLAG_OSD OPTIMIZE_FLAG_MASK(0)
#define OPTIMIZE_FLAG_CPD OPTIMIZE_FLAG_MASK(1)
#define OPTIMIZE_FLAG_OPK OPTIMIZE_FLAG_MASK(2)
#define OPTIMIZE_FLAG_SET_MASK(val, mask) (val) |= (mask)
#define OPTIMIZE_FLAG_TEST_MASK(val, mask) (((val) & (mask)) != 0)
typedef struct SOptimizeContext {
SPlanContext* pPlanCxt;
bool optimized;
} SOptimizeContext;
@ -57,7 +60,23 @@ typedef enum ECondAction {
// after supporting outer join, there are other possibilities
} ECondAction;
EDealRes haveNormalColImpl(SNode* pNode, void* pContext) {
typedef bool (*FMayBeOptimized)(SLogicNode* pNode);
static SLogicNode* optFindPossibleNode(SLogicNode* pNode, FMayBeOptimized func) {
if (func(pNode)) {
return pNode;
}
SNode* pChild;
FOREACH(pChild, pNode->pChildren) {
SLogicNode* pScanNode = optFindPossibleNode((SLogicNode*)pChild, func);
if (NULL != pScanNode) {
return pScanNode;
}
}
return NULL;
}
EDealRes osdHaveNormalColImpl(SNode* pNode, void* pContext) {
if (QUERY_NODE_COLUMN == nodeType(pNode)) {
*((bool*)pContext) = (COLUMN_TYPE_TAG != ((SColumnNode*)pNode)->colType);
return *((bool*)pContext) ? DEAL_RES_END : DEAL_RES_IGNORE_CHILD;
@ -65,9 +84,9 @@ EDealRes haveNormalColImpl(SNode* pNode, void* pContext) {
return DEAL_RES_CONTINUE;
}
static bool haveNormalCol(SNodeList* pList) {
static bool osdHaveNormalCol(SNodeList* pList) {
bool res = false;
nodesWalkExprsPostOrder(pList, haveNormalColImpl, &res);
nodesWalkExprsPostOrder(pList, osdHaveNormalColImpl, &res);
return res;
}
@ -89,21 +108,7 @@ static bool osdMayBeOptimized(SLogicNode* pNode) {
if (QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent)) {
return (WINDOW_TYPE_INTERVAL == ((SWindowLogicNode*)pNode->pParent)->winType);
}
return !haveNormalCol(((SAggLogicNode*)pNode->pParent)->pGroupKeys);
}
static SLogicNode* osdFindPossibleScanNode(SLogicNode* pNode) {
if (osdMayBeOptimized(pNode)) {
return pNode;
}
SNode* pChild;
FOREACH(pChild, pNode->pChildren) {
SLogicNode* pScanNode = osdFindPossibleScanNode((SLogicNode*)pChild);
if (NULL != pScanNode) {
return pScanNode;
}
}
return NULL;
return !osdHaveNormalCol(((SAggLogicNode*)pNode->pParent)->pGroupKeys);
}
static SNodeList* osdGetAllFuncs(SLogicNode* pNode) {
@ -138,7 +143,7 @@ static int32_t osdGetRelatedFuncs(SScanLogicNode* pScan, SNodeList** pSdrFuncs,
}
static int32_t osdMatch(SOptimizeContext* pCxt, SLogicNode* pLogicNode, SOsdInfo* pInfo) {
pInfo->pScan = (SScanLogicNode*)osdFindPossibleScanNode(pLogicNode);
pInfo->pScan = (SScanLogicNode*)optFindPossibleNode(pLogicNode, osdMayBeOptimized);
if (NULL == pInfo->pScan) {
return TSDB_CODE_SUCCESS;
}
@ -345,7 +350,7 @@ static int32_t cpdCalcTimeRange(SScanLogicNode* pScan, SNode** pPrimaryKeyCond,
}
static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode* pScan) {
if (NULL == pScan->node.pConditions) {
if (NULL == pScan->node.pConditions || OPTIMIZE_FLAG_TEST_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_CPD)) {
return TSDB_CODE_SUCCESS;
}
@ -359,7 +364,10 @@ static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode*
pScan->node.pConditions = pOtherCond;
}
if (TSDB_CODE_SUCCESS != code) {
if (TSDB_CODE_SUCCESS == code) {
OPTIMIZE_FLAG_SET_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_CPD);
pCxt->optimized = true;
} else {
nodesDestroyNode(pPrimaryKeyCond);
nodesDestroyNode(pOtherCond);
}
@ -367,7 +375,7 @@ static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode*
return code;
}
static bool belongThisTable(SNode* pCondCol, SNodeList* pTableCols) {
static bool cpdBelongThisTable(SNode* pCondCol, SNodeList* pTableCols) {
SNode* pTableCol = NULL;
FOREACH(pTableCol, pTableCols) {
if (nodesEqualNode(pCondCol, pTableCol)) {
@ -380,9 +388,9 @@ static bool belongThisTable(SNode* pCondCol, SNodeList* pTableCols) {
static EDealRes cpdIsMultiTableCondImpl(SNode* pNode, void* pContext) {
SCpdIsMultiTableCondCxt* pCxt = pContext;
if (QUERY_NODE_COLUMN == nodeType(pNode)) {
if (belongThisTable(pNode, pCxt->pLeftCols)) {
if (cpdBelongThisTable(pNode, pCxt->pLeftCols)) {
pCxt->havaLeftCol = true;
} else if (belongThisTable(pNode, pCxt->pRightCols)) {
} else if (cpdBelongThisTable(pNode, pCxt->pRightCols)) {
pCxt->haveRightCol = true;
}
return pCxt->havaLeftCol && pCxt->haveRightCol ? DEAL_RES_END : DEAL_RES_CONTINUE;
@ -508,11 +516,76 @@ static int32_t cpdPushCondToChild(SOptimizeContext* pCxt, SLogicNode* pChild, SN
return TSDB_CODE_PLAN_INTERNAL_ERROR;
}
static bool cpdIsPrimaryKey(SNode* pNode, SNodeList* pTableCols) {
if (QUERY_NODE_COLUMN != nodeType(pNode)) {
return false;
}
SColumnNode* pCol = (SColumnNode*)pNode;
if (PRIMARYKEY_TIMESTAMP_COL_ID != pCol->colId) {
return false;
}
return cpdBelongThisTable(pNode, pTableCols);
}
static bool cpdIsPrimaryKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) {
if (QUERY_NODE_OPERATOR != nodeType(pCond)) {
return false;
}
SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets;
SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets;
SOperatorNode* pOper = (SOperatorNode*)pJoin->pOnConditions;
if (cpdIsPrimaryKey(pOper->pLeft, pLeftCols)) {
return cpdIsPrimaryKey(pOper->pRight, pRightCols);
} else if (cpdIsPrimaryKey(pOper->pLeft, pRightCols)) {
return cpdIsPrimaryKey(pOper->pRight, pLeftCols);
}
return false;
}
static int32_t cpdCheckOpCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SNode* pOnCond) {
if (!cpdIsPrimaryKeyEqualCond(pJoin, pOnCond)) {
snprintf(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, "l.ts = r.ts is expected in join expression");
return TSDB_CODE_FAILED;
}
return TSDB_CODE_SUCCESS;
}
static int32_t cpdCheckLogicCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SLogicConditionNode* pOnCond) {
if (LOGIC_COND_TYPE_AND != pOnCond->condType) {
snprintf(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, "l.ts = r.ts is expected in join expression");
return TSDB_CODE_FAILED;
}
SNode* pCond = NULL;
FOREACH(pCond, pOnCond->pParameterList) {
if (!cpdIsPrimaryKeyEqualCond(pJoin, pCond)) {
snprintf(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, "l.ts = r.ts is expected in join expression");
return TSDB_CODE_FAILED;
}
}
return TSDB_CODE_SUCCESS;
}
static int32_t cpdCheckJoinOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) {
if (NULL == pJoin->pOnConditions) {
snprintf(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, "not support cross join");
return TSDB_CODE_FAILED;
}
if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOnConditions)) {
return cpdCheckLogicCond(pCxt, pJoin, (SLogicConditionNode*)pJoin->pOnConditions);
} else {
return cpdCheckOpCond(pCxt, pJoin, pJoin->pOnConditions);
}
}
static int32_t cpdPushJoinCondition(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) {
if (NULL == pJoin->node.pConditions) {
if (OPTIMIZE_FLAG_TEST_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_CPD)) {
return TSDB_CODE_SUCCESS;
}
if (NULL == pJoin->node.pConditions) {
return cpdCheckJoinOnCond(pCxt, pJoin);
}
SNode* pOnCond = NULL;
SNode* pLeftChildCond = NULL;
SNode* pRightChildCond = NULL;
@ -527,7 +600,11 @@ static int32_t cpdPushJoinCondition(SOptimizeContext* pCxt, SJoinLogicNode* pJoi
code = cpdPushCondToChild(pCxt, (SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1), &pRightChildCond);
}
if (TSDB_CODE_SUCCESS != code) {
if (TSDB_CODE_SUCCESS == code) {
OPTIMIZE_FLAG_SET_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_CPD);
pCxt->optimized = true;
code = cpdCheckJoinOnCond(pCxt, pJoin);
} else {
nodesDestroyNode(pOnCond);
nodesDestroyNode(pLeftChildCond);
nodesDestroyNode(pRightChildCond);
@ -572,15 +649,129 @@ static int32_t cpdOptimize(SOptimizeContext* pCxt, SLogicNode* pLogicNode) {
return cpdPushCondition(pCxt, pLogicNode);
}
static bool opkIsPrimaryKeyOrderBy(SNodeList* pSortKeys) {
if (1 != LIST_LENGTH(pSortKeys)) {
return false;
}
SNode* pNode = ((SOrderByExprNode*)nodesListGetNode(pSortKeys, 0))->pExpr;
return (QUERY_NODE_COLUMN == nodeType(pNode) ? (PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pNode)->colId) : false);
}
static bool opkSortMayBeOptimized(SLogicNode* pNode) {
if (QUERY_NODE_LOGIC_PLAN_SORT != nodeType(pNode)) {
return false;
}
if (OPTIMIZE_FLAG_TEST_MASK(pNode->optimizedFlag, OPTIMIZE_FLAG_OPK)) {
return false;
}
return true;
}
static int32_t opkGetScanNodesImpl(SLogicNode* pNode, bool* pNotOptimize, SNodeList** pScanNodes) {
int32_t code = TSDB_CODE_SUCCESS;
switch (nodeType(pNode)) {
case QUERY_NODE_LOGIC_PLAN_SCAN:
return nodesListMakeAppend(pScanNodes, pNode);
case QUERY_NODE_LOGIC_PLAN_JOIN:
code = opkGetScanNodesImpl(nodesListGetNode(pNode->pChildren, 0), pNotOptimize, pScanNodes);
if (TSDB_CODE_SUCCESS == code) {
code = opkGetScanNodesImpl(nodesListGetNode(pNode->pChildren, 1), pNotOptimize, pScanNodes);
}
return code;
case QUERY_NODE_LOGIC_PLAN_AGG:
*pNotOptimize = true;
return code;
default:
break;
}
if (1 != LIST_LENGTH(pNode->pChildren)) {
*pNotOptimize = true;
}
return opkGetScanNodesImpl(nodesListGetNode(pNode->pChildren, 0), pNotOptimize, pScanNodes);
}
static int32_t opkGetScanNodes(SLogicNode* pNode, SNodeList** pScanNodes) {
bool notOptimize = false;
int32_t code = opkGetScanNodesImpl(pNode, &notOptimize, pScanNodes);
if (TSDB_CODE_SUCCESS != code || notOptimize) {
nodesClearList(*pScanNodes);
}
return code;
}
static EOrder opkGetPrimaryKeyOrder(SSortLogicNode* pSort) {
return ((SOrderByExprNode*)nodesListGetNode(pSort->pSortKeys, 0))->order;
}
static SNode* opkRewriteDownNode(SSortLogicNode* pSort) {
SNode* pDownNode = nodesListGetNode(pSort->node.pChildren, 0);
// todo
pSort->node.pChildren = NULL;
return pDownNode;
}
static int32_t opkDoOptimized(SOptimizeContext* pCxt, SSortLogicNode* pSort, SNodeList* pScanNodes) {
EOrder order = opkGetPrimaryKeyOrder(pSort);
if (ORDER_DESC == order) {
SNode* pScan = NULL;
FOREACH(pScan, pScanNodes) {
((SScanLogicNode*)pScan)->scanSeq[0] = 0;
((SScanLogicNode*)pScan)->scanSeq[1] = 1;
}
}
if (NULL == pSort->node.pParent) {
// todo
return TSDB_CODE_SUCCESS;
}
SNode* pDownNode = opkRewriteDownNode(pSort);
SNode* pNode;
FOREACH(pNode, pSort->node.pParent->pChildren) {
if (nodesEqualNode(pNode, pSort)) {
REPLACE_NODE(pDownNode);
break;
}
}
nodesDestroyNode(pSort);
return TSDB_CODE_SUCCESS;
}
static int32_t opkOptimizeImpl(SOptimizeContext* pCxt, SSortLogicNode* pSort) {
OPTIMIZE_FLAG_SET_MASK(pSort->node.optimizedFlag, OPTIMIZE_FLAG_OPK);
if (!opkIsPrimaryKeyOrderBy(pSort->pSortKeys) || 1 != LIST_LENGTH(pSort->node.pChildren)) {
return TSDB_CODE_SUCCESS;
}
SNodeList* pScanNodes = NULL;
int32_t code = opkGetScanNodes(nodesListGetNode(pSort->node.pChildren, 0), &pScanNodes);
if (TSDB_CODE_SUCCESS == code && NULL != pScanNodes) {
code = opkDoOptimized(pCxt, pSort, pScanNodes);
}
nodesClearList(pScanNodes);
return code;
}
static int32_t opkOptimize(SOptimizeContext* pCxt, SLogicNode* pLogicNode) {
SSortLogicNode* pSort = (SSortLogicNode*)optFindPossibleNode(pLogicNode, opkSortMayBeOptimized);
if (NULL == pSort) {
return TSDB_CODE_SUCCESS;
}
return opkOptimizeImpl(pCxt, pSort);
}
static const SOptimizeRule optimizeRuleSet[] = {
{ .pName = "OptimizeScanData", .optimizeFunc = osdOptimize },
{ .pName = "ConditionPushDown", .optimizeFunc = cpdOptimize }
{ .pName = "ConditionPushDown", .optimizeFunc = cpdOptimize },
{ .pName = "OrderByPrimaryKey", .optimizeFunc = opkOptimize }
};
static const int32_t optimizeRuleNum = (sizeof(optimizeRuleSet) / sizeof(SOptimizeRule));
static int32_t applyOptimizeRule(SLogicNode* pLogicNode) {
SOptimizeContext cxt = { .optimized = false };
static int32_t applyOptimizeRule(SPlanContext* pCxt, SLogicNode* pLogicNode) {
SOptimizeContext cxt = { .pPlanCxt = pCxt, .optimized = false };
do {
cxt.optimized = false;
for (int32_t i = 0; i < optimizeRuleNum; ++i) {
@ -594,5 +785,5 @@ static int32_t applyOptimizeRule(SLogicNode* pLogicNode) {
}
int32_t optimizeLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode) {
return applyOptimizeRule(pLogicNode);
return applyOptimizeRule(pCxt, pLogicNode);
}

5
source/libs/planner/src/planPhysiCreater.c
View File

@ -398,9 +398,6 @@ static int32_t createScanPhysiNodeFinalize(SPhysiPlanContext* pCxt, SScanLogicNo
if (TSDB_CODE_SUCCESS == code) {
pScanPhysiNode->uid = pScanLogicNode->pMeta->uid;
pScanPhysiNode->tableType = pScanLogicNode->pMeta->tableType;
pScanPhysiNode->order = TSDB_ORDER_ASC;
pScanPhysiNode->count = 1;
pScanPhysiNode->reverse = 0;
memcpy(&pScanPhysiNode->tableName, &pScanLogicNode->tableName, sizeof(SName));
}
@ -432,7 +429,7 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp
return TSDB_CODE_OUT_OF_MEMORY;
}
pTableScan->scanFlag = pScanLogicNode->scanFlag;
memcpy(pTableScan->scanSeq, pScanLogicNode->scanSeq, sizeof(pScanLogicNode->scanSeq));
pTableScan->scanRange = pScanLogicNode->scanRange;
pTableScan->ratio = pScanLogicNode->ratio;
vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode);

23
tools/shell/src/backup/tnettest.h → source/libs/planner/test/planOptTest.cpp
View File

@ -13,17 +13,20 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TNETTEST_H
#define TDENGINE_TNETTEST_H
#include "planTestUtil.h"
#include "planner.h"
#ifdef __cplusplus
extern "C" {
#endif
using namespace std;
void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, int32_t pkgNum, char *pkgType);
class PlanOptimizeTest : public PlannerTestBase {
#ifdef __cplusplus
};
TEST_F(PlanOptimizeTest, orderByPrimaryKey) {
useDb("root", "test");
run("select * from t1 order by ts");
run("select * from t1 order by ts desc");
run("select c1 from t1 order by ts");
run("select c1 from t1 order by ts desc");
}
#endif
#endif // TDENGINE_TNETTEST_H

249
source/libs/qworker/inc/qworkerInt.h
View File

@ -25,10 +25,10 @@ extern "C" {
#include "ttimer.h"
#define QW_DEFAULT_SCHEDULER_NUMBER 10000
#define QW_DEFAULT_TASK_NUMBER 10000
#define QW_DEFAULT_SCH_TASK_NUMBER 10000
#define QW_DEFAULT_SHORT_RUN_TIMES 2
#define QW_DEFAULT_HEARTBEAT_MSEC 3000
#define QW_DEFAULT_TASK_NUMBER 10000
#define QW_DEFAULT_SCH_TASK_NUMBER 10000
#define QW_DEFAULT_SHORT_RUN_TIMES 2
#define QW_DEFAULT_HEARTBEAT_MSEC 3000
enum {
QW_PHASE_PRE_QUERY = 1,
@ -60,7 +60,6 @@ enum {
QW_WRITE,
};
enum {
QW_NOT_EXIST_RET_ERR = 1,
QW_NOT_EXIST_ADD,
@ -73,25 +72,26 @@ typedef struct SQWDebug {
} SQWDebug;
typedef struct SQWConnInfo {
void *handle;
void *ahandle;
void * handle;
void * ahandle;
int64_t refId;
} SQWConnInfo;
typedef struct SQWMsg {
void *node;
int32_t code;
char *msg;
int32_t msgLen;
SQWConnInfo connInfo;
void * node;
int32_t code;
char * msg;
int32_t msgLen;
SQWConnInfo connInfo;
} SQWMsg;
typedef struct SQWHbInfo {
SSchedulerHbRsp rsp;
SQWConnInfo connInfo;
SSchedulerHbRsp rsp;
SQWConnInfo connInfo;
} SQWHbInfo;
typedef struct SQWPhaseInput {
int32_t code;
int32_t code;
} SQWPhaseInput;
typedef struct SQWPhaseOutput {
@ -100,41 +100,40 @@ typedef struct SQWPhaseOutput {
#endif
} SQWPhaseOutput;
typedef struct SQWTaskStatus {
int64_t refId; // job's refId
int32_t code;
int8_t status;
typedef struct SQWTaskStatus {
int64_t refId; // job's refId
int32_t code;
int8_t status;
} SQWTaskStatus;
typedef struct SQWTaskCtx {
SRWLatch lock;
int8_t phase;
int8_t taskType;
int8_t explain;
bool queryFetched;
bool queryEnd;
bool queryContinue;
bool queryInQueue;
int32_t rspCode;
SRWLatch lock;
int8_t phase;
int8_t taskType;
int8_t explain;
SQWConnInfo ctrlConnInfo;
SQWConnInfo dataConnInfo;
bool queryFetched;
bool queryEnd;
bool queryContinue;
bool queryInQueue;
int32_t rspCode;
int8_t events[QW_EVENT_MAX];
qTaskInfo_t taskHandle;
DataSinkHandle sinkHandle;
SQWConnInfo ctrlConnInfo;
SQWConnInfo dataConnInfo;
int8_t events[QW_EVENT_MAX];
void *taskHandle;
void *sinkHandle;
} SQWTaskCtx;
typedef struct SQWSchStatus {
int32_t lastAccessTs; // timestamp in second
int32_t lastAccessTs; // timestamp in second
SRWLatch hbConnLock;
SQWConnInfo hbConnInfo;
SQueryNodeEpId hbEpId;
SQueryNodeEpId hbEpId;
SRWLatch tasksLock;
SHashObj *tasksHash; // key:queryId+taskId, value: SQWTaskStatus
SHashObj * tasksHash; // key:queryId+taskId, value: SQWTaskStatus
} SQWSchStatus;
// Qnode/Vnode level task management
@ -142,100 +141,146 @@ typedef struct SQWorkerMgmt {
SQWorkerCfg cfg;
int8_t nodeType;
int32_t nodeId;
void *timer;
void * timer;
tmr_h hbTimer;
SRWLatch schLock;
// SRWLatch ctxLock;
SHashObj *schHash; // key: schedulerId, value: SQWSchStatus
SHashObj *ctxHash; // key: queryId+taskId, value: SQWTaskCtx
SMsgCb msgCb;
SHashObj *schHash; // key: schedulerId, value: SQWSchStatus
SHashObj *ctxHash; // key: queryId+taskId, value: SQWTaskCtx
SMsgCb msgCb;
} SQWorkerMgmt;
#define QW_FPARAMS_DEF SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int64_t rId
#define QW_IDS() sId, qId, tId, rId
#define QW_FPARAMS() mgmt, QW_IDS()
#define QW_IDS() sId, qId, tId, rId
#define QW_FPARAMS() mgmt, QW_IDS()
#define QW_GET_EVENT_VALUE(ctx, event) atomic_load_8(&(ctx)->events[event])
#define QW_IS_EVENT_RECEIVED(ctx, event) (atomic_load_8(&(ctx)->events[event]) == QW_EVENT_RECEIVED)
#define QW_IS_EVENT_PROCESSED(ctx, event) (atomic_load_8(&(ctx)->events[event]) == QW_EVENT_PROCESSED)
#define QW_SET_EVENT_RECEIVED(ctx, event) atomic_store_8(&(ctx)->events[event], QW_EVENT_RECEIVED)
#define QW_IS_EVENT_RECEIVED(ctx, event) (atomic_load_8(&(ctx)->events[event]) == QW_EVENT_RECEIVED)
#define QW_IS_EVENT_PROCESSED(ctx, event) (atomic_load_8(&(ctx)->events[event]) == QW_EVENT_PROCESSED)
#define QW_SET_EVENT_RECEIVED(ctx, event) atomic_store_8(&(ctx)->events[event], QW_EVENT_RECEIVED)
#define QW_SET_EVENT_PROCESSED(ctx, event) atomic_store_8(&(ctx)->events[event], QW_EVENT_PROCESSED)
#define QW_GET_PHASE(ctx) atomic_load_8(&(ctx)->phase)
#define QW_SET_RSP_CODE(ctx, code) atomic_store_32(&(ctx)->rspCode, code)
#define QW_SET_RSP_CODE(ctx, code) atomic_store_32(&(ctx)->rspCode, code)
#define QW_UPDATE_RSP_CODE(ctx, code) atomic_val_compare_exchange_32(&(ctx)->rspCode, 0, code)
#define QW_IS_QUERY_RUNNING(ctx) (QW_GET_PHASE(ctx) == QW_PHASE_PRE_QUERY || QW_GET_PHASE(ctx) == QW_PHASE_PRE_CQUERY)
#define QW_TASK_NOT_EXIST(code) (TSDB_CODE_QRY_SCH_NOT_EXIST == (code) || TSDB_CODE_QRY_TASK_NOT_EXIST == (code))
#define QW_TASK_NOT_EXIST(code) (TSDB_CODE_QRY_SCH_NOT_EXIST == (code) || TSDB_CODE_QRY_TASK_NOT_EXIST == (code))
#define QW_TASK_ALREADY_EXIST(code) (TSDB_CODE_QRY_TASK_ALREADY_EXIST == (code))
#define QW_TASK_READY(status) (status == JOB_TASK_STATUS_SUCCEED || status == JOB_TASK_STATUS_FAILED || status == JOB_TASK_STATUS_CANCELLED || status == JOB_TASK_STATUS_PARTIAL_SUCCEED)
#define QW_SET_QTID(id, qId, tId) do { *(uint64_t *)(id) = (qId); *(uint64_t *)((char *)(id) + sizeof(qId)) = (tId); } while (0)
#define QW_GET_QTID(id, qId, tId) do { (qId) = *(uint64_t *)(id); (tId) = *(uint64_t *)((char *)(id) + sizeof(qId)); } while (0)
#define QW_TASK_READY(status) \
(status == JOB_TASK_STATUS_SUCCEED || status == JOB_TASK_STATUS_FAILED || status == JOB_TASK_STATUS_CANCELLED || \
status == JOB_TASK_STATUS_PARTIAL_SUCCEED)
#define QW_SET_QTID(id, qId, tId) \
do { \
*(uint64_t *)(id) = (qId); \
*(uint64_t *)((char *)(id) + sizeof(qId)) = (tId); \
} while (0)
#define QW_GET_QTID(id, qId, tId) \
do { \
(qId) = *(uint64_t *)(id); \
(tId) = *(uint64_t *)((char *)(id) + sizeof(qId)); \
} while (0)
#define QW_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
#define QW_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
#define QW_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
#define QW_ERR_RET(c) \
do { \
int32_t _code = c; \
if (_code != TSDB_CODE_SUCCESS) { \
terrno = _code; \
return _code; \
} \
} while (0)
#define QW_RET(c) \
do { \
int32_t _code = c; \
if (_code != TSDB_CODE_SUCCESS) { \
terrno = _code; \
} \
return _code; \
} while (0)
#define QW_ERR_JRET(c) \
do { \
code = c; \
if (code != TSDB_CODE_SUCCESS) { \
terrno = code; \
goto _return; \
} \
} while (0)
#define QW_ELOG(param, ...) qError("QW:%p " param, mgmt, __VA_ARGS__)
#define QW_DLOG(param, ...) qDebug("QW:%p " param, mgmt, __VA_ARGS__)
#define QW_DUMP(param, ...) do { if (gQWDebug.dumpEnable) { qDebug("QW:%p " param, mgmt, __VA_ARGS__); } } while (0)
#define QW_DUMP(param, ...) \
do { \
if (gQWDebug.dumpEnable) { \
qDebug("QW:%p " param, mgmt, __VA_ARGS__); \
} \
} while (0)
#define QW_SCH_ELOG(param, ...) qError("QW:%p SID:%" PRIx64 " " param, mgmt, sId, __VA_ARGS__)
#define QW_SCH_DLOG(param, ...) qDebug("QW:%p SID:%" PRIx64 " " param, mgmt, sId, __VA_ARGS__)
#define QW_SCH_ELOG(param, ...) qError("QW:%p SID:%"PRIx64" " param, mgmt, sId, __VA_ARGS__)
#define QW_SCH_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64" " param, mgmt, sId, __VA_ARGS__)
#define QW_TASK_ELOG(param, ...) qError("QW:%p QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, mgmt, qId, tId, __VA_ARGS__)
#define QW_TASK_WLOG(param, ...) qWarn("QW:%p QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, mgmt, qId, tId, __VA_ARGS__)
#define QW_TASK_DLOG(param, ...) qDebug("QW:%p QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, mgmt, qId, tId, __VA_ARGS__)
#define QW_TASK_DLOGL(param, ...) \
qDebugL("QW:%p QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, mgmt, qId, tId, __VA_ARGS__)
#define QW_TASK_ELOG(param, ...) qError("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__)
#define QW_TASK_WLOG(param, ...) qWarn("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__)
#define QW_TASK_DLOG(param, ...) qDebug("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__)
#define QW_TASK_DLOGL(param, ...) qDebugL("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__)
#define QW_TASK_ELOG_E(param) qError("QW:%p QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, mgmt, qId, tId)
#define QW_TASK_WLOG_E(param) qWarn("QW:%p QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, mgmt, qId, tId)
#define QW_TASK_DLOG_E(param) qDebug("QW:%p QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, mgmt, qId, tId)
#define QW_TASK_ELOG_E(param) qError("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId)
#define QW_TASK_WLOG_E(param) qWarn("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId)
#define QW_TASK_DLOG_E(param) qDebug("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId)
#define QW_SCH_TASK_ELOG(param, ...) \
qError("QW:%p SID:0x%" PRIx64 ",QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, mgmt, sId, qId, tId, __VA_ARGS__)
#define QW_SCH_TASK_WLOG(param, ...) \
qWarn("QW:%p SID:0x%" PRIx64 ",QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, mgmt, sId, qId, tId, __VA_ARGS__)
#define QW_SCH_TASK_DLOG(param, ...) \
qDebug("QW:%p SID:0x%" PRIx64 ",QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, mgmt, sId, qId, tId, __VA_ARGS__)
#define QW_SCH_TASK_ELOG(param, ...) qError("QW:%p SID:0x%"PRIx64",QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__)
#define QW_SCH_TASK_WLOG(param, ...) qWarn("QW:%p SID:0x%"PRIx64",QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__)
#define QW_SCH_TASK_DLOG(param, ...) qDebug("QW:%p SID:0x%"PRIx64",QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__)
#define QW_LOCK_DEBUG(...) do { if (gQWDebug.lockEnable) { qDebug(__VA_ARGS__); } } while (0)
#define QW_LOCK_DEBUG(...) \
do { \
if (gQWDebug.lockEnable) { \
qDebug(__VA_ARGS__); \
} \
} while (0)
#define TD_RWLATCH_WRITE_FLAG_COPY 0x40000000
#define QW_LOCK(type, _lock) do { \
if (QW_READ == (type)) { \
assert(atomic_load_32((_lock)) >= 0); \
QW_LOCK_DEBUG("QW RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
taosRLockLatch(_lock); \
QW_LOCK_DEBUG("QW RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
assert(atomic_load_32((_lock)) > 0); \
} else { \
assert(atomic_load_32((_lock)) >= 0); \
QW_LOCK_DEBUG("QW WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
taosWLockLatch(_lock); \
QW_LOCK_DEBUG("QW WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \
} \
} while (0)
#define QW_UNLOCK(type, _lock) do { \
if (QW_READ == (type)) { \
assert(atomic_load_32((_lock)) > 0); \
QW_LOCK_DEBUG("QW RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
taosRUnLockLatch(_lock); \
QW_LOCK_DEBUG("QW RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
assert(atomic_load_32((_lock)) >= 0); \
} else { \
assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \
QW_LOCK_DEBUG("QW WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
taosWUnLockLatch(_lock); \
QW_LOCK_DEBUG("QW WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
assert(atomic_load_32((_lock)) >= 0); \
} \
} while (0)
#define QW_LOCK(type, _lock) \
do { \
if (QW_READ == (type)) { \
assert(atomic_load_32((_lock)) >= 0); \
QW_LOCK_DEBUG("QW RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
taosRLockLatch(_lock); \
QW_LOCK_DEBUG("QW RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
assert(atomic_load_32((_lock)) > 0); \
} else { \
assert(atomic_load_32((_lock)) >= 0); \
QW_LOCK_DEBUG("QW WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
taosWLockLatch(_lock); \
QW_LOCK_DEBUG("QW WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \
} \
} while (0)
#define QW_UNLOCK(type, _lock) \
do { \
if (QW_READ == (type)) { \
assert(atomic_load_32((_lock)) > 0); \
QW_LOCK_DEBUG("QW RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
taosRUnLockLatch(_lock); \
QW_LOCK_DEBUG("QW RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
assert(atomic_load_32((_lock)) >= 0); \
} else { \
assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \
QW_LOCK_DEBUG("QW WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
taosWUnLockLatch(_lock); \
QW_LOCK_DEBUG("QW WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
assert(atomic_load_32((_lock)) >= 0); \
} \
} while (0)
#ifdef __cplusplus
}

653
source/libs/qworker/src/qworker.c
View File

File diff suppressed because it is too large Load Diff

290
source/libs/qworker/src/qworkerMsg.c
View File

@ -1,18 +1,17 @@
#include "qworker.h"
#include "tcommon.h"
#include "qworkerMsg.h"
#include "dataSinkMgt.h"
#include "executor.h"
#include "planner.h"
#include "query.h"
#include "qworker.h"
#include "qworkerInt.h"
#include "qworkerMsg.h"
#include "tcommon.h"
#include "tmsg.h"
#include "tname.h"
#include "dataSinkMgt.h"
int32_t qwMallocFetchRsp(int32_t length, SRetrieveTableRsp **rsp) {
int32_t msgSize = sizeof(SRetrieveTableRsp) + length;
SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)rpcMallocCont(msgSize);
if (NULL == pRsp) {
qError("rpcMallocCont %d failed", msgSize);
@ -26,11 +25,9 @@ int32_t qwMallocFetchRsp(int32_t length, SRetrieveTableRsp **rsp) {
return TSDB_CODE_SUCCESS;
}
void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, bool qComplete) {
SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)msg;
rsp->useconds = htobe64(input->useconds);
rsp->completed = qComplete;
rsp->precision = input->precision;
@ -39,7 +36,6 @@ void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, bool qComplete)
rsp->numOfRows = htonl(input->numOfRows);
}
void qwFreeFetchRsp(void *msg) {
if (msg) {
rpcFreeCont(msg);
@ -48,18 +44,19 @@ void qwFreeFetchRsp(void *msg) {
int32_t qwBuildAndSendQueryRsp(SQWConnInfo *pConn, int32_t code) {
SQueryTableRsp rsp = {.code = code};
int32_t contLen = tSerializeSQueryTableRsp(NULL, 0, &rsp);
void *msg = rpcMallocCont(contLen);
void * msg = rpcMallocCont(contLen);
tSerializeSQueryTableRsp(msg, contLen, &rsp);
SRpcMsg rpcRsp = {
.msgType = TDMT_VND_QUERY_RSP,
.handle = pConn->handle,
.ahandle = pConn->ahandle,
.pCont = msg,
.contLen = contLen,
.code = code,
.msgType = TDMT_VND_QUERY_RSP,
.handle = pConn->handle,
.ahandle = pConn->ahandle,
.refId = pConn->refId,
.pCont = msg,
.contLen = contLen,
.code = code,
};
tmsgSendRsp(&rpcRsp);
@ -72,12 +69,13 @@ int32_t qwBuildAndSendReadyRsp(SQWConnInfo *pConn, int32_t code) {
pRsp->code = code;
SRpcMsg rpcRsp = {
.msgType = TDMT_VND_RES_READY_RSP,
.handle = pConn->handle,
.ahandle = NULL,
.pCont = pRsp,
.contLen = sizeof(*pRsp),
.code = code,
.msgType = TDMT_VND_RES_READY_RSP,
.handle = pConn->handle,
.refId = pConn->refId,
.ahandle = NULL,
.pCont = pRsp,
.contLen = sizeof(*pRsp),
.code = code,
};
tmsgSendRsp(&rpcRsp);
@ -85,20 +83,21 @@ int32_t qwBuildAndSendReadyRsp(SQWConnInfo *pConn, int32_t code) {
return TSDB_CODE_SUCCESS;
}
int32_t qwBuildAndSendExplainRsp(SQWConnInfo *pConn, SExplainExecInfo *execInfo, int32_t num) {
int32_t qwBuildAndSendExplainRsp(SQWConnInfo *pConn, SExplainExecInfo *execInfo, int32_t num) {
SExplainRsp rsp = {.numOfPlans = num, .subplanInfo = execInfo};
int32_t contLen = tSerializeSExplainRsp(NULL, 0, &rsp);
void *pRsp = rpcMallocCont(contLen);
void * pRsp = rpcMallocCont(contLen);
tSerializeSExplainRsp(pRsp, contLen, &rsp);
SRpcMsg rpcRsp = {
.msgType = TDMT_VND_EXPLAIN_RSP,
.handle = pConn->handle,
.ahandle = pConn->ahandle,
.pCont = pRsp,
.contLen = contLen,
.code = 0,
.msgType = TDMT_VND_EXPLAIN_RSP,
.handle = pConn->handle,
.ahandle = pConn->ahandle,
.refId = pConn->refId,
.pCont = pRsp,
.contLen = contLen,
.code = 0,
};
tmsgSendRsp(&rpcRsp);
@ -108,16 +107,17 @@ int32_t qwBuildAndSendExplainRsp(SQWConnInfo *pConn, SExplainExecInfo *execInfo,
int32_t qwBuildAndSendHbRsp(SQWConnInfo *pConn, SSchedulerHbRsp *pStatus, int32_t code) {
int32_t contLen = tSerializeSSchedulerHbRsp(NULL, 0, pStatus);
void *pRsp = rpcMallocCont(contLen);
void * pRsp = rpcMallocCont(contLen);
tSerializeSSchedulerHbRsp(pRsp, contLen, pStatus);
SRpcMsg rpcRsp = {
.msgType = TDMT_VND_QUERY_HEARTBEAT_RSP,
.handle = pConn->handle,
.ahandle = pConn->ahandle,
.pCont = pRsp,
.contLen = contLen,
.code = code,
.msgType = TDMT_VND_QUERY_HEARTBEAT_RSP,
.handle = pConn->handle,
.ahandle = pConn->ahandle,
.refId = pConn->refId,
.pCont = pRsp,
.contLen = contLen,
.code = code,
};
tmsgSendRsp(&rpcRsp);
@ -133,12 +133,13 @@ int32_t qwBuildAndSendFetchRsp(SQWConnInfo *pConn, SRetrieveTableRsp *pRsp, int3
}
SRpcMsg rpcRsp = {
.msgType = TDMT_VND_FETCH_RSP,
.handle = pConn->handle,
.ahandle = pConn->ahandle,
.pCont = pRsp,
.contLen = sizeof(*pRsp) + dataLength,
.code = code,
.msgType = TDMT_VND_FETCH_RSP,
.handle = pConn->handle,
.ahandle = pConn->ahandle,
.refId = pConn->refId,
.pCont = pRsp,
.contLen = sizeof(*pRsp) + dataLength,
.code = code,
};
tmsgSendRsp(&rpcRsp);
@ -151,12 +152,13 @@ int32_t qwBuildAndSendCancelRsp(SQWConnInfo *pConn, int32_t code) {
pRsp->code = code;
SRpcMsg rpcRsp = {
.msgType = TDMT_VND_CANCEL_TASK_RSP,
.handle = pConn->handle,
.ahandle = pConn->ahandle,
.pCont = pRsp,
.contLen = sizeof(*pRsp),
.code = code,
.msgType = TDMT_VND_CANCEL_TASK_RSP,
.handle = pConn->handle,
.ahandle = pConn->ahandle,
.refId = pConn->refId,
.pCont = pRsp,
.contLen = sizeof(*pRsp),
.code = code,
};
tmsgSendRsp(&rpcRsp);
@ -168,12 +170,13 @@ int32_t qwBuildAndSendDropRsp(SQWConnInfo *pConn, int32_t code) {
pRsp->code = code;
SRpcMsg rpcRsp = {
.msgType = TDMT_VND_DROP_TASK_RSP,
.handle = pConn->handle,
.ahandle = pConn->ahandle,
.pCont = pRsp,
.contLen = sizeof(*pRsp),
.code = code,
.msgType = TDMT_VND_DROP_TASK_RSP,
.handle = pConn->handle,
.ahandle = pConn->ahandle,
.refId = pConn->refId,
.pCont = pRsp,
.contLen = sizeof(*pRsp),
.code = code,
};
tmsgSendRsp(&rpcRsp);
@ -191,7 +194,7 @@ int32_t qwBuildAndSendShowRsp(SRpcMsg *pMsg, int32_t code) {
return -1;
}
col_id_t cols = 0;
col_id_t cols = 0;
SSchema *pSchema = showRsp.tableMeta.pSchemas;
const SSchema *s = tGetTbnameColumnSchema();
@ -220,12 +223,13 @@ int32_t qwBuildAndSendShowRsp(SRpcMsg *pMsg, int32_t code) {
showRsp.tableMeta.numOfColumns = cols;
int32_t bufLen = tSerializeSShowRsp(NULL, 0, &showRsp);
void *pBuf = rpcMallocCont(bufLen);
void * pBuf = rpcMallocCont(bufLen);
tSerializeSShowRsp(pBuf, bufLen, &showRsp);
SRpcMsg rpcMsg = {
.handle = pMsg->handle,
.ahandle = pMsg->ahandle,
.refId = pMsg->refId,
.pCont = pBuf,
.contLen = bufLen,
.code = code,
@ -235,17 +239,18 @@ int32_t qwBuildAndSendShowRsp(SRpcMsg *pMsg, int32_t code) {
return TSDB_CODE_SUCCESS;
}
int32_t qwBuildAndSendShowFetchRsp(SRpcMsg *pMsg, SVShowTablesFetchReq* pFetchReq) {
int32_t qwBuildAndSendShowFetchRsp(SRpcMsg *pMsg, SVShowTablesFetchReq *pFetchReq) {
SVShowTablesFetchRsp *pRsp = (SVShowTablesFetchRsp *)rpcMallocCont(sizeof(SVShowTablesFetchRsp));
int32_t handle = htonl(pFetchReq->id);
int32_t handle = htonl(pFetchReq->id);
pRsp->numOfRows = 0;
SRpcMsg rpcMsg = {
.handle = pMsg->handle,
.handle = pMsg->handle,
.ahandle = pMsg->ahandle,
.pCont = pRsp,
.refId = pMsg->refId,
.pCont = pRsp,
.contLen = sizeof(*pRsp),
.code = 0,
.code = 0,
};
tmsgSendRsp(&rpcMsg);
@ -253,7 +258,7 @@ int32_t qwBuildAndSendShowFetchRsp(SRpcMsg *pMsg, SVShowTablesFetchReq* pFetchRe
}
int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, SQWConnInfo *pConn) {
SQueryContinueReq * req = (SQueryContinueReq *)rpcMallocCont(sizeof(SQueryContinueReq));
SQueryContinueReq *req = (SQueryContinueReq *)rpcMallocCont(sizeof(SQueryContinueReq));
if (NULL == req) {
QW_SCH_TASK_ELOG("rpcMallocCont %d failed", (int32_t)sizeof(SQueryContinueReq));
QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
@ -265,12 +270,13 @@ int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, SQWConnInfo *pConn) {
req->taskId = tId;
SRpcMsg pNewMsg = {
.handle = pConn->handle,
.ahandle = pConn->ahandle,
.msgType = TDMT_VND_QUERY_CONTINUE,
.pCont = req,
.contLen = sizeof(SQueryContinueReq),
.code = 0,
.handle = pConn->handle,
.ahandle = pConn->ahandle,
.msgType = TDMT_VND_QUERY_CONTINUE,
.refId = pConn->refId,
.pCont = req,
.contLen = sizeof(SQueryContinueReq),
.code = 0,
};
int32_t code = tmsgPutToQueue(&mgmt->msgCb, QUERY_QUEUE, &pNewMsg);
@ -285,29 +291,29 @@ int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, SQWConnInfo *pConn) {
return TSDB_CODE_SUCCESS;
}
int32_t qwRegisterQueryBrokenLinkArg(QW_FPARAMS_DEF, SQWConnInfo *pConn) {
STaskDropReq * req = (STaskDropReq *)rpcMallocCont(sizeof(STaskDropReq));
STaskDropReq *req = (STaskDropReq *)rpcMallocCont(sizeof(STaskDropReq));
if (NULL == req) {
QW_SCH_TASK_ELOG("rpcMallocCont %d failed", (int32_t)sizeof(STaskDropReq));
QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
}
req->header.vgId = htonl(mgmt->nodeId);
req->sId = htobe64(sId);
req->queryId = htobe64(qId);
req->taskId = htobe64(tId);
req->refId = htobe64(rId);
SRpcMsg pMsg = {
.handle = pConn->handle,
.ahandle = pConn->ahandle,
.msgType = TDMT_VND_DROP_TASK,
.pCont = req,
.contLen = sizeof(STaskDropReq),
.code = TSDB_CODE_RPC_NETWORK_UNAVAIL,
.handle = pConn->handle,
.ahandle = pConn->ahandle,
.refId = pConn->refId,
.msgType = TDMT_VND_DROP_TASK,
.pCont = req,
.contLen = sizeof(STaskDropReq),
.code = TSDB_CODE_RPC_NETWORK_UNAVAIL,
};
tmsgRegisterBrokenLinkArg(&mgmt->msgCb, &pMsg);
return TSDB_CODE_SUCCESS;
@ -333,43 +339,42 @@ int32_t qwRegisterHbBrokenLinkArg(SQWorkerMgmt *mgmt, uint64_t sId, SQWConnInfo
taosMemoryFree(msg);
QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
SRpcMsg pMsg = {
.handle = pConn->handle,
.ahandle = pConn->ahandle,
.msgType = TDMT_VND_QUERY_HEARTBEAT,
.pCont = msg,
.contLen = sizeof(SSchedulerHbReq),
.code = TSDB_CODE_RPC_NETWORK_UNAVAIL,
.handle = pConn->handle,
.ahandle = pConn->ahandle,
.refId = pConn->refId,
.msgType = TDMT_VND_QUERY_HEARTBEAT,
.pCont = msg,
.contLen = msgSize,
.code = TSDB_CODE_RPC_NETWORK_UNAVAIL,
};
tmsgRegisterBrokenLinkArg(&mgmt->msgCb, &pMsg);
return TSDB_CODE_SUCCESS;
}
int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) {
QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
}
int32_t code = 0;
int32_t code = 0;
SSubQueryMsg *msg = pMsg->pCont;
SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt;
if (NULL == msg || pMsg->contLen <= sizeof(*msg)) {
QW_ELOG("invalid query msg, msg:%p, msgLen:%d", msg, pMsg->contLen);
QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
}
msg->sId = be64toh(msg->sId);
msg->sId = be64toh(msg->sId);
msg->queryId = be64toh(msg->queryId);
msg->taskId = be64toh(msg->taskId);
msg->refId = be64toh(msg->refId);
msg->phyLen = ntohl(msg->phyLen);
msg->sqlLen = ntohl(msg->sqlLen);
msg->taskId = be64toh(msg->taskId);
msg->refId = be64toh(msg->refId);
msg->phyLen = ntohl(msg->phyLen);
msg->sqlLen = ntohl(msg->sqlLen);
uint64_t sId = msg->sId;
uint64_t qId = msg->queryId;
@ -379,8 +384,9 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
SQWMsg qwMsg = {.node = node, .msg = msg->msg + msg->sqlLen, .msgLen = msg->phyLen};
qwMsg.connInfo.handle = pMsg->handle;
qwMsg.connInfo.ahandle = pMsg->ahandle;
qwMsg.connInfo.refId = pMsg->refId;
char* sql = strndup(msg->msg, msg->sqlLen);
char *sql = strndup(msg->msg, msg->sqlLen);
QW_SCH_TASK_DLOG("processQuery start, node:%p, handle:%p, sql:%s", node, pMsg->handle, sql);
taosMemoryFreeClear(sql);
@ -388,17 +394,17 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
QW_SCH_TASK_DLOG("processQuery end, node:%p", node);
return TSDB_CODE_SUCCESS;
return TSDB_CODE_SUCCESS;
}
int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
int32_t code = 0;
int8_t status = 0;
bool queryDone = false;
int32_t code = 0;
int8_t status = 0;
bool queryDone = false;
SQueryContinueReq *msg = (SQueryContinueReq *)pMsg->pCont;
bool needStop = false;
SQWTaskCtx *handles = NULL;
SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt;
bool needStop = false;
SQWTaskCtx * handles = NULL;
SQWorkerMgmt * mgmt = (SQWorkerMgmt *)qWorkerMgmt;
if (NULL == msg || pMsg->contLen < sizeof(*msg)) {
QW_ELOG("invalid cquery msg, msg:%p, msgLen:%d", msg, pMsg->contLen);
@ -408,11 +414,12 @@ int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
uint64_t sId = msg->sId;
uint64_t qId = msg->queryId;
uint64_t tId = msg->taskId;
int64_t rId = 0;
int64_t rId = 0;
SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0};
qwMsg.connInfo.handle = pMsg->handle;
qwMsg.connInfo.ahandle = pMsg->ahandle;
qwMsg.connInfo.refId = pMsg->refId;
QW_SCH_TASK_DLOG("processCQuery start, node:%p, handle:%p", node, pMsg->handle);
@ -420,10 +427,10 @@ int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
QW_SCH_TASK_DLOG("processCQuery end, node:%p", node);
return TSDB_CODE_SUCCESS;
return TSDB_CODE_SUCCESS;
}
int32_t qWorkerProcessReadyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg){
int32_t qWorkerProcessReadyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) {
return TSDB_CODE_QRY_INVALID_INPUT;
}
@ -433,7 +440,7 @@ int32_t qWorkerProcessReadyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg){
if (NULL == msg || pMsg->contLen < sizeof(*msg)) {
QW_ELOG("invalid task ready msg, msg:%p, msgLen:%d", msg, pMsg->contLen);
QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
}
}
msg->sId = be64toh(msg->sId);
msg->queryId = be64toh(msg->queryId);
@ -442,18 +449,19 @@ int32_t qWorkerProcessReadyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg){
uint64_t sId = msg->sId;
uint64_t qId = msg->queryId;
uint64_t tId = msg->taskId;
int64_t rId = 0;
int64_t rId = 0;
SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0};
qwMsg.connInfo.handle = pMsg->handle;
qwMsg.connInfo.ahandle = pMsg->ahandle;
qwMsg.connInfo.refId = pMsg->refId;
QW_SCH_TASK_DLOG("processReady start, node:%p, handle:%p", node, pMsg->handle);
QW_ERR_RET(qwProcessReady(QW_FPARAMS(), &qwMsg));
QW_SCH_TASK_DLOG("processReady end, node:%p", node);
return TSDB_CODE_SUCCESS;
}
@ -462,24 +470,24 @@ int32_t qWorkerProcessStatusMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
return TSDB_CODE_QRY_INVALID_INPUT;
}
int32_t code = 0;
int32_t code = 0;
SSchTasksStatusReq *msg = pMsg->pCont;
if (NULL == msg || pMsg->contLen < sizeof(*msg)) {
qError("invalid task status msg");
QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
}
}
SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt;
msg->sId = htobe64(msg->sId);
uint64_t sId = msg->sId;
SSchedulerStatusRsp *sStatus = NULL;
//QW_ERR_JRET(qwGetSchTasksStatus(qWorkerMgmt, msg->sId, &sStatus));
// QW_ERR_JRET(qwGetSchTasksStatus(qWorkerMgmt, msg->sId, &sStatus));
_return:
//QW_ERR_RET(qwBuildAndSendStatusRsp(pMsg, sStatus));
// QW_ERR_RET(qwBuildAndSendStatusRsp(pMsg, sStatus));
return TSDB_CODE_SUCCESS;
}
@ -491,11 +499,11 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
SResFetchReq *msg = pMsg->pCont;
SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt;
if (NULL == msg || pMsg->contLen < sizeof(*msg)) {
QW_ELOG("invalid fetch msg, msg:%p, msgLen:%d", msg, pMsg->contLen);
QW_ELOG("invalid fetch msg, msg:%p, msgLen:%d", msg, pMsg->contLen);
QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
}
}
msg->sId = be64toh(msg->sId);
msg->queryId = be64toh(msg->queryId);
@ -504,11 +512,12 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
uint64_t sId = msg->sId;
uint64_t qId = msg->queryId;
uint64_t tId = msg->taskId;
int64_t rId = 0;
int64_t rId = 0;
SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0};
qwMsg.connInfo.handle = pMsg->handle;
qwMsg.connInfo.ahandle = pMsg->ahandle;
qwMsg.connInfo.refId = pMsg->refId;
QW_SCH_TASK_DLOG("processFetch start, node:%p, handle:%p", node, pMsg->handle);
@ -516,7 +525,7 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
QW_SCH_TASK_DLOG("processFetch end, node:%p", node);
return TSDB_CODE_SUCCESS;
return TSDB_CODE_SUCCESS;
}
int32_t qWorkerProcessFetchRsp(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
@ -529,13 +538,13 @@ int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
return TSDB_CODE_QRY_INVALID_INPUT;
}
SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt;
int32_t code = 0;
SQWorkerMgmt * mgmt = (SQWorkerMgmt *)qWorkerMgmt;
int32_t code = 0;
STaskCancelReq *msg = pMsg->pCont;
if (NULL == msg || pMsg->contLen < sizeof(*msg)) {
qError("invalid task cancel msg");
qError("invalid task cancel msg");
QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
}
}
msg->sId = be64toh(msg->sId);
msg->queryId = be64toh(msg->queryId);
@ -550,8 +559,9 @@ int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0};
qwMsg.connInfo.handle = pMsg->handle;
qwMsg.connInfo.ahandle = pMsg->ahandle;
qwMsg.connInfo.refId = pMsg->refId;
//QW_ERR_JRET(qwCancelTask(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId));
// QW_ERR_JRET(qwCancelTask(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId));
_return:
@ -566,14 +576,14 @@ int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
return TSDB_CODE_QRY_INVALID_INPUT;
}
int32_t code = 0;
int32_t code = 0;
STaskDropReq *msg = pMsg->pCont;
SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt;
if (NULL == msg || pMsg->contLen < sizeof(*msg)) {
QW_ELOG("invalid task drop msg, msg:%p, msgLen:%d", msg, pMsg->contLen);
QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
}
}
msg->sId = be64toh(msg->sId);
msg->queryId = be64toh(msg->queryId);
@ -588,9 +598,10 @@ int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0, .code = pMsg->code};
qwMsg.connInfo.handle = pMsg->handle;
qwMsg.connInfo.ahandle = pMsg->ahandle;
qwMsg.connInfo.refId = pMsg->refId;
if (TSDB_CODE_RPC_NETWORK_UNAVAIL == pMsg->code) {
QW_SCH_TASK_DLOG("receive drop task due to network broken, error:%s", tstrerror(pMsg->code));
QW_SCH_TASK_DLOG("receive drop task due to network broken, error:%s", tstrerror(pMsg->code));
}
QW_SCH_TASK_DLOG("processDrop start, node:%p, handle:%p", node, pMsg->handle);
@ -607,14 +618,14 @@ int32_t qWorkerProcessHbMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
return TSDB_CODE_QRY_INVALID_INPUT;
}
int32_t code = 0;
int32_t code = 0;
SSchedulerHbReq req = {0};
SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt;
SQWorkerMgmt * mgmt = (SQWorkerMgmt *)qWorkerMgmt;
if (NULL == pMsg->pCont) {
QW_ELOG("invalid hb msg, msg:%p, msgLen:%d", pMsg->pCont, pMsg->contLen);
QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
}
}
if (tDeserializeSSchedulerHbReq(pMsg->pCont, pMsg->contLen, &req)) {
QW_ELOG("invalid hb msg, msg:%p, msgLen:%d", pMsg->pCont, pMsg->contLen);
@ -623,12 +634,13 @@ int32_t qWorkerProcessHbMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
}
uint64_t sId = req.sId;
SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0, .code = pMsg->code};
SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0, .code = pMsg->code};
qwMsg.connInfo.handle = pMsg->handle;
qwMsg.connInfo.ahandle = pMsg->ahandle;
qwMsg.connInfo.refId = pMsg->refId;
if (TSDB_CODE_RPC_NETWORK_UNAVAIL == pMsg->code) {
QW_SCH_DLOG("receive Hb msg due to network broken, error:%s", tstrerror(pMsg->code));
QW_SCH_DLOG("receive Hb msg due to network broken, error:%s", tstrerror(pMsg->code));
}
QW_SCH_DLOG("processHb start, node:%p, handle:%p", node, pMsg->handle);
@ -645,7 +657,7 @@ int32_t qWorkerProcessShowMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
return TSDB_CODE_QRY_INVALID_INPUT;
}
int32_t code = 0;
int32_t code = 0;
SVShowTablesReq *pReq = pMsg->pCont;
QW_RET(qwBuildAndSendShowRsp(pMsg, code));
}

64
source/libs/scalar/src/sclfunc.c
View File

@ -32,8 +32,8 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
float *in = (float *)pInputData->pData;
float *out = (float *)pOutputData->pData;
for (int32_t i = 0; i < pInput->numOfRows; ++i) {
if (colDataIsNull_f(pInputData->nullbitmap, i)) {
colDataSetNull_f(pOutputData->nullbitmap, i);
if (colDataIsNull_s(pInputData, i)) {
colDataAppendNULL(pOutputData, i);
continue;
}
out[i] = (in[i] >= 0)? in[i] : -in[i];
@ -45,8 +45,8 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
double *in = (double *)pInputData->pData;
double *out = (double *)pOutputData->pData;
for (int32_t i = 0; i < pInput->numOfRows; ++i) {
if (colDataIsNull_f(pInputData->nullbitmap, i)) {
colDataSetNull_f(pOutputData->nullbitmap, i);
if (colDataIsNull_s(pInputData, i)) {
colDataAppendNULL(pOutputData, i);
continue;
}
out[i] = (in[i] >= 0)? in[i] : -in[i];
@ -58,8 +58,8 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
int8_t *in = (int8_t *)pInputData->pData;
int8_t *out = (int8_t *)pOutputData->pData;
for (int32_t i = 0; i < pInput->numOfRows; ++i) {
if (colDataIsNull_f(pInputData->nullbitmap, i)) {
colDataSetNull_f(pOutputData->nullbitmap, i);
if (colDataIsNull_s(pInputData, i)) {
colDataAppendNULL(pOutputData, i);
continue;
}
out[i] = (in[i] >= 0)? in[i] : -in[i];
@ -71,8 +71,8 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
int16_t *in = (int16_t *)pInputData->pData;
int16_t *out = (int16_t *)pOutputData->pData;
for (int32_t i = 0; i < pInput->numOfRows; ++i) {
if (colDataIsNull_f(pInputData->nullbitmap, i)) {
colDataSetNull_f(pOutputData->nullbitmap, i);
if (colDataIsNull_s(pInputData, i)) {
colDataAppendNULL(pOutputData, i);
continue;
}
out[i] = (in[i] >= 0)? in[i] : -in[i];
@ -84,8 +84,8 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
int32_t *in = (int32_t *)pInputData->pData;
int32_t *out = (int32_t *)pOutputData->pData;
for (int32_t i = 0; i < pInput->numOfRows; ++i) {
if (colDataIsNull_f(pInputData->nullbitmap, i)) {
colDataSetNull_f(pOutputData->nullbitmap, i);
if (colDataIsNull_s(pInputData, i)) {
colDataAppendNULL(pOutputData, i);
continue;
}
out[i] = (in[i] >= 0)? in[i] : -in[i];
@ -97,8 +97,8 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
int64_t *in = (int64_t *)pInputData->pData;
int64_t *out = (int64_t *)pOutputData->pData;
for (int32_t i = 0; i < pInput->numOfRows; ++i) {
if (colDataIsNull_f(pInputData->nullbitmap, i)) {
colDataSetNull_f(pOutputData->nullbitmap, i);
if (colDataIsNull_s(pInputData, i)) {
colDataAppendNULL(pOutputData, i);
continue;
}
out[i] = (in[i] >= 0)? in[i] : -in[i];
@ -129,8 +129,8 @@ static int32_t doScalarFunctionUnique(SScalarParam *pInput, int32_t inputNum, SS
double *out = (double *)pOutputData->pData;
for (int32_t i = 0; i < pInput->numOfRows; ++i) {
if (colDataIsNull_f(pInputData->nullbitmap, i)) {
colDataSetNull_f(pOutputData->nullbitmap, i);
if (colDataIsNull_s(pInputData, i)) {
colDataAppendNULL(pOutputData, i);
continue;
}
out[i] = valFn(getValueFn(pInputData->pData, i));
@ -157,9 +157,9 @@ static int32_t doScalarFunctionUnique2(SScalarParam *pInput, int32_t inputNum, S
double *out = (double *)pOutputData->pData;
for (int32_t i = 0; i < pInput->numOfRows; ++i) {
if (colDataIsNull_f(pInputData[0]->nullbitmap, i) ||
colDataIsNull_f(pInputData[1]->nullbitmap, 0)) {
colDataSetNull_f(pOutputData->nullbitmap, i);
if (colDataIsNull_s(pInputData[0], i) ||
colDataIsNull_s(pInputData[1], 0)) {
colDataAppendNULL(pOutputData, i);
continue;
}
out[i] = valFn(getValueFn[0](pInputData[0]->pData, i), getValueFn[1](pInputData[1]->pData, 0));
@ -184,8 +184,8 @@ static int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
float *out = (float *)pOutputData->pData;
for (int32_t i = 0; i < pInput->numOfRows; ++i) {
if (colDataIsNull_f(pInputData->nullbitmap, i)) {
colDataSetNull_f(pOutputData->nullbitmap, i);
if (colDataIsNull_s(pInputData, i)) {
colDataAppendNULL(pOutputData, i);
continue;
}
out[i] = f1(in[i]);
@ -198,8 +198,8 @@ static int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
double *out = (double *)pOutputData->pData;
for (int32_t i = 0; i < pInput->numOfRows; ++i) {
if (colDataIsNull_f(pInputData->nullbitmap, i)) {
colDataSetNull_f(pOutputData->nullbitmap, i);
if (colDataIsNull_s(pInputData, i)) {
colDataAppendNULL(pOutputData, i);
continue;
}
out[i] = d1(in[i]);
@ -301,7 +301,7 @@ static int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
for (int32_t i = 0; i < pInput->numOfRows; ++i) {
if (colDataIsNull_s(pInputData, i)) {
colDataSetNull_f(pOutputData->nullbitmap, i);
colDataAppendNULL(pOutputData, i);
continue;
}
@ -1247,26 +1247,28 @@ int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
}
int32_t nowFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
if (inputNum != 1) {
return TSDB_CODE_FAILED;
int64_t ts = taosGetTimestamp(TSDB_TIME_PRECISION_MILLI);
for (int32_t i = 0; i < pInput->numOfRows; ++i) {
colDataAppendInt64(pOutput->columnData, i, &ts);
}
colDataAppendInt64(pOutput->columnData, pOutput->numOfRows, (int64_t *)colDataGetData(pInput->columnData, 0));
pOutput->numOfRows = pInput->numOfRows;
return TSDB_CODE_SUCCESS;
}
int32_t todayFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
if (inputNum != 1) {
return TSDB_CODE_FAILED;
int64_t ts = taosGetTimestampToday(TSDB_TIME_PRECISION_MILLI);
for (int32_t i = 0; i < pInput->numOfRows; ++i) {
colDataAppendInt64(pOutput->columnData, i, &ts);
}
colDataAppendInt64(pOutput->columnData, pOutput->numOfRows, (int64_t *)colDataGetData(pInput->columnData, 0));
pOutput->numOfRows = pInput->numOfRows;
return TSDB_CODE_SUCCESS;
}
int32_t timezoneFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
if (inputNum != 1) {
return TSDB_CODE_FAILED;
for (int32_t i = 0; i < pInput->numOfRows; ++i) {
colDataAppend(pOutput->columnData, i, tsTimezoneStr, false);
}
colDataAppend(pOutput->columnData, pOutput->numOfRows, (char *)colDataGetData(pInput->columnData, 0), false);
pOutput->numOfRows = pInput->numOfRows;
return TSDB_CODE_SUCCESS;
}

26
source/libs/scheduler/src/scheduler.c
View File

@ -369,7 +369,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) {
}
for (int32_t n = 0; n < childNum; ++n) {
SSubplan *child = (SSubplan *)nodesListGetNode(pPlan->pChildren, n);
SSubplan * child = (SSubplan *)nodesListGetNode(pPlan->pChildren, n);
SSchTask **childTask = taosHashGet(planToTask, &child, POINTER_BYTES);
if (NULL == childTask || NULL == *childTask) {
SCH_TASK_ELOG("subplan children relationship error, level:%d, taskIdx:%d, childIdx:%d", i, m, n);
@ -401,7 +401,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) {
}
for (int32_t n = 0; n < parentNum; ++n) {
SSubplan *parent = (SSubplan *)nodesListGetNode(pPlan->pParents, n);
SSubplan * parent = (SSubplan *)nodesListGetNode(pPlan->pParents, n);
SSchTask **parentTask = taosHashGet(planToTask, &parent, POINTER_BYTES);
if (NULL == parentTask || NULL == *parentTask) {
SCH_TASK_ELOG("subplan parent relationship error, level:%d, taskIdx:%d, childIdx:%d", i, m, n);
@ -491,7 +491,7 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) {
SSchLevel level = {0};
SNodeListNode *plans = NULL;
int32_t taskNum = 0;
SSchLevel *pLevel = NULL;
SSchLevel * pLevel = NULL;
level.status = JOB_TASK_STATUS_NOT_START;
@ -1267,7 +1267,7 @@ int32_t schUpdateTaskExecNodeHandle(SSchTask *pTask, void *handle, int32_t rspCo
int32_t schHandleCallback(void *param, const SDataBuf *pMsg, int32_t msgType, int32_t rspCode) {
int32_t code = 0;
SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param;
SSchTask *pTask = NULL;
SSchTask * pTask = NULL;
SSchJob *pJob = schAcquireJob(pParam->refId);
if (NULL == pJob) {
@ -1617,8 +1617,8 @@ _return:
int32_t schMakeHbRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) {
int32_t code = 0;
SSchHbCallbackParam *param = NULL;
SMsgSendInfo *pMsgSendInfo = NULL;
SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx);
SMsgSendInfo * pMsgSendInfo = NULL;
SQueryNodeAddr * addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx);
SQueryNodeEpId epId = {0};
epId.nodeId = addr->nodeId;
@ -1759,10 +1759,10 @@ int32_t schCloneHbRpcCtx(SRpcCtx *pSrc, SRpcCtx *pDst) {
}
SRpcCtxVal dst = {0};
void *pIter = taosHashIterate(pSrc->args, NULL);
void * pIter = taosHashIterate(pSrc->args, NULL);
while (pIter) {
SRpcCtxVal *pVal = (SRpcCtxVal *)pIter;
int32_t *msgType = taosHashGetKey(pIter, NULL);
int32_t * msgType = taosHashGetKey(pIter, NULL);
dst = *pVal;
dst.val = NULL;
@ -1916,7 +1916,7 @@ _return:
int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t msgType) {
uint32_t msgSize = 0;
void *msg = NULL;
void * msg = NULL;
int32_t code = 0;
bool isCandidateAddr = false;
bool persistHandle = false;
@ -2673,7 +2673,7 @@ int32_t schedulerGetTasksStatus(int64_t job, SArray *pSub) {
SSchLevel *pLevel = taosArrayGet(pJob->levels, i);
for (int32_t m = 0; m < pLevel->taskNum; ++m) {
SSchTask *pTask = taosArrayGet(pLevel->subTasks, m);
SSchTask * pTask = taosArrayGet(pLevel->subTasks, m);
SQuerySubDesc subDesc = {.tid = pTask->taskId, .status = pTask->status};
taosArrayPush(pSub, &subDesc);
@ -2734,7 +2734,7 @@ void schedulerFreeTaskList(SArray *taskList) {
void schedulerDestroy(void) {
if (schMgmt.jobRef) {
SSchJob *pJob = taosIterateRef(schMgmt.jobRef, 0);
int64_t refId = 0;
int64_t refId = 0;
while (pJob) {
refId = pJob->refId;
@ -2751,12 +2751,12 @@ void schedulerDestroy(void) {
}
if (schMgmt.hbConnections) {
void *pIter = taosHashIterate(schMgmt.hbConnections, NULL);
void *pIter = taosHashIterate(schMgmt.hbConnections, NULL);
while (pIter != NULL) {
SSchHbTrans *hb = pIter;
schFreeRpcCtx(&hb->rpcCtx);
pIter = taosHashIterate(schMgmt.hbConnections, pIter);
}
}
taosHashCleanup(schMgmt.hbConnections);
schMgmt.hbConnections = NULL;
}

3
source/libs/sync/src/syncIO.c
View File

@ -170,7 +170,6 @@ static int32_t syncIOStartInternal(SSyncIO *io) {
taosBlockSIGPIPE();
rpcInit();
tsRpcForceTcp = 1;
// cient rpc init
{
@ -448,4 +447,4 @@ static void syncIOTickPing(void *param, void *tmrId) {
syncPingDestroy(pMsg);
taosTmrReset(syncIOTickPing, io->pingTimerMS, io, io->timerMgr, &io->pingTimer);
}
}

4
source/libs/transport/inc/transComm.h
View File

@ -327,6 +327,10 @@ void transQueueClear(STransQueue* queue);
*/
void transQueueDestroy(STransQueue* queue);
/*
* init global func
*/
void transThreadOnce();
#ifdef __cplusplus
}
#endif

9
source/libs/transport/src/transComm.c
View File

@ -16,6 +16,8 @@
#include "transComm.h"
// static TdThreadOnce transModuleInit = PTHREAD_ONCE_INIT;
int transAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey) {
T_MD5_CTX context;
int ret = -1;
@ -361,5 +363,10 @@ void transQueueDestroy(STransQueue* queue) {
transQueueClear(queue);
taosArrayDestroy(queue->q);
}
// int32_t transGetExHandle() {
// static
//}
// void transThreadOnce() {
// taosThreadOnce(&transModuleInit, );
//}
#endif

265
source/libs/transport/src/transSrv.c
View File

@ -17,6 +17,11 @@
#include "transComm.h"
static TdThreadOnce transModuleInit = PTHREAD_ONCE_INIT;
static char* notify = "a";
static int transSrvInst = 0;
typedef struct {
int notifyCount; //
int init; // init or not
@ -45,11 +50,12 @@ typedef struct SSrvConn {
struct sockaddr_in addr;
struct sockaddr_in locaddr;
int spi;
char info[64];
char user[TSDB_UNI_LEN]; // user ID for the link
char secret[TSDB_PASSWORD_LEN];
char ckey[TSDB_PASSWORD_LEN]; // ciphering key
int64_t refId;
int spi;
char info[64];
char user[TSDB_UNI_LEN]; // user ID for the link
char secret[TSDB_PASSWORD_LEN];
char ckey[TSDB_PASSWORD_LEN]; // ciphering key
} SSrvConn;
typedef struct SSrvMsg {
@ -89,7 +95,12 @@ typedef struct SServerObj {
uv_async_t* pAcceptAsync; // just to quit from from accept thread
} SServerObj;
static const char* notify = "a";
// handle
typedef struct SExHandle {
void* handle;
int64_t refId;
SWorkThrdObj* pThrd;
} SExHandle;
static void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf);
static void uvAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf);
@ -130,6 +141,17 @@ static void uvHandleRegister(SSrvMsg* msg, SWorkThrdObj* thrd);
static void (*transAsyncHandle[])(SSrvMsg* msg, SWorkThrdObj* thrd) = {uvHandleResp, uvHandleQuit, uvHandleRelease,
uvHandleRegister};
static int32_t exHandlesMgt;
void uvInitExHandleMgt();
void uvOpenExHandleMgt(int size);
void uvCloseExHandleMgt();
int64_t uvAddExHandle(void* p);
int32_t uvRemoveExHandle(int64_t refId);
int32_t uvReleaseExHandle(int64_t refId);
void uvDestoryExHandle(void* handle);
SExHandle* uvAcquireExHandle(int64_t refId);
static void uvDestroyConn(uv_handle_t* handle);
// server and worker thread
@ -168,6 +190,40 @@ static bool addHandleToAcceptloop(void* arg);
uv_loop_close(loop); \
} while (0);
#define ASYNC_ERR_JRET(thrd) \
do { \
if (thrd->quit) { \
tTrace("worker thread already quit, ignore msg"); \
goto _return1; \
} \
} while (0)
#define ASYNC_CHECK_HANDLE(exh1, refId) \
do { \
if (refId > 0) { \
tTrace("server handle step1"); \
SExHandle* exh2 = uvAcquireExHandle(refId); \
if (exh2 == NULL || refId != exh2->refId) { \
tTrace("server handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, \
exh1->refId, refId); \
goto _return1; \
} \
} else if (refId == 0) { \
tTrace("server handle step2"); \
SExHandle* exh2 = uvAcquireExHandle(refId); \
if (exh2 == NULL || refId != exh2->refId) { \
tTrace("server handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, \
refId, exh2 ? exh2->refId : 0); \
goto _return1; \
} else { \
refId = exh1->refId; \
} \
} else if (refId == -1) { \
tTrace("server handle step3"); \
goto _return2; \
} \
} while (0)
void uvAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) {
SSrvConn* conn = handle->data;
SConnBuffer* pBuf = &conn->readBuf;
@ -233,7 +289,15 @@ static void uvHandleReq(SSrvConn* pConn) {
// 1. server application should not send resp on handle
// 2. once send out data, cli conn released to conn pool immediately
// 3. not mixed with persist
transMsg.handle = pConn;
transMsg.handle = (void*)uvAcquireExHandle(pConn->refId);
tTrace("server handle %p conn: %p translated to app, refId: %" PRIu64 "", transMsg.handle, pConn, pConn->refId);
transMsg.refId = pConn->refId;
assert(transMsg.handle != NULL);
if (pHead->noResp == 1) {
transMsg.refId = -1;
}
uvReleaseExHandle(pConn->refId);
STrans* pTransInst = pConn->pTransInst;
(*pTransInst->cfp)(pTransInst->parent, &transMsg, NULL);
@ -404,6 +468,7 @@ static void destroySmsg(SSrvMsg* smsg) {
taosMemoryFree(smsg);
}
static void destroyAllConn(SWorkThrdObj* pThrd) {
tTrace("thread %p destroy all conn ", pThrd);
while (!QUEUE_IS_EMPTY(&pThrd->conn)) {
queue* h = QUEUE_HEAD(&pThrd->conn);
QUEUE_REMOVE(h);
@ -436,7 +501,26 @@ void uvWorkerAsyncCb(uv_async_t* handle) {
tError("unexcept occurred, continue");
continue;
}
(*transAsyncHandle[msg->type])(msg, pThrd);
// release handle to rpc init
if (msg->type == Quit) {
(*transAsyncHandle[msg->type])(msg, pThrd);
continue;
} else {
STransMsg transMsg = msg->msg;
SExHandle* exh1 = transMsg.handle;
int64_t refId = transMsg.refId;
SExHandle* exh2 = uvAcquireExHandle(refId);
if (exh2 == NULL || exh1 != exh2) {
tTrace("server handle %p except msg, ignore it", exh1);
uvReleaseExHandle(refId);
destroySmsg(msg);
continue;
}
msg->pConn = exh1->handle;
uvReleaseExHandle(refId);
(*transAsyncHandle[msg->type])(msg, pThrd);
}
}
}
static void uvWalkCb(uv_handle_t* handle, void* arg) {
@ -658,8 +742,15 @@ static SSrvConn* createConn(void* hThrd) {
pConn->broken = false;
pConn->status = ConnNormal;
SExHandle* exh = taosMemoryMalloc(sizeof(SExHandle));
exh->handle = pConn;
exh->pThrd = pThrd;
exh->refId = uvAddExHandle(exh);
uvAcquireExHandle(exh->refId);
pConn->refId = exh->refId;
transRefSrvHandle(pConn);
tTrace("server conn %p created", pConn);
tTrace("server handle %p, conn %p created, refId: %" PRId64 "", exh, pConn, pConn->refId);
return pConn;
}
@ -667,6 +758,7 @@ static void destroyConn(SSrvConn* conn, bool clear) {
if (conn == NULL) {
return;
}
transDestroyBuffer(&conn->readBuf);
if (clear) {
tTrace("server conn %p to be destroyed", conn);
@ -681,6 +773,9 @@ static void uvDestroyConn(uv_handle_t* handle) {
}
SWorkThrdObj* thrd = conn->hostThrd;
uvReleaseExHandle(conn->refId);
uvRemoveExHandle(conn->refId);
tDebug("server conn %p destroy", conn);
// uv_timer_stop(&conn->pTimer);
transQueueDestroy(&conn->srvMsgs);
@ -705,6 +800,10 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads,
srv->port = port;
uv_loop_init(srv->loop);
taosThreadOnce(&transModuleInit, uvInitExHandleMgt);
transSrvInst++;
// uvOpenExHandleMgt(10000);
for (int i = 0; i < srv->numOfThreads; i++) {
SWorkThrdObj* thrd = (SWorkThrdObj*)taosMemoryCalloc(1, sizeof(SWorkThrdObj));
thrd->quit = false;
@ -749,6 +848,44 @@ End:
transCloseServer(srv);
return NULL;
}
void uvInitExHandleMgt() {
// init exhandle mgt
uvOpenExHandleMgt(10000);
}
void uvOpenExHandleMgt(int size) {
// added into once later
exHandlesMgt = taosOpenRef(size, uvDestoryExHandle);
}
void uvCloseExHandleMgt() {
// close ref
taosCloseRef(exHandlesMgt);
}
int64_t uvAddExHandle(void* p) {
// acquire extern handle
return taosAddRef(exHandlesMgt, p);
}
int32_t uvRemoveExHandle(int64_t refId) {
// acquire extern handle
return taosRemoveRef(exHandlesMgt, refId);
}
SExHandle* uvAcquireExHandle(int64_t refId) {
// acquire extern handle
return (SExHandle*)taosAcquireRef(exHandlesMgt, refId);
}
int32_t uvReleaseExHandle(int64_t refId) {
// release extern handle
return taosReleaseRef(exHandlesMgt, refId);
}
void uvDestoryExHandle(void* handle) {
if (handle == NULL) {
return;
}
taosMemoryFree(handle);
}
void uvHandleQuit(SSrvMsg* msg, SWorkThrdObj* thrd) {
thrd->quit = true;
if (QUEUE_IS_EMPTY(&thrd->conn)) {
@ -759,7 +896,6 @@ void uvHandleQuit(SSrvMsg* msg, SWorkThrdObj* thrd) {
taosMemoryFree(msg);
}
void uvHandleRelease(SSrvMsg* msg, SWorkThrdObj* thrd) {
// release handle to rpc init
SSrvConn* conn = msg->pConn;
if (conn->status == ConnAcquire) {
if (!transQueuePush(&conn->srvMsgs, msg)) {
@ -840,6 +976,12 @@ void transCloseServer(void* arg) {
taosMemoryFree(srv->pipe);
taosMemoryFree(srv);
transSrvInst--;
if (transSrvInst == 0) {
transModuleInit = PTHREAD_ONCE_INIT;
uvCloseExHandleMgt();
}
}
void transRefSrvHandle(void* handle) {
@ -862,57 +1004,98 @@ void transUnrefSrvHandle(void* handle) {
}
void transReleaseSrvHandle(void* handle) {
if (handle == NULL) {
return;
}
SSrvConn* pConn = handle;
SWorkThrdObj* pThrd = pConn->hostThrd;
SExHandle* exh = handle;
int64_t refId = exh->refId;
STransMsg tmsg = {.code = 0, .handle = handle, .ahandle = NULL};
ASYNC_CHECK_HANDLE(exh, refId);
SWorkThrdObj* pThrd = exh->pThrd;
ASYNC_ERR_JRET(pThrd);
STransMsg tmsg = {.code = 0, .handle = exh, .ahandle = NULL, .refId = refId};
SSrvMsg* srvMsg = taosMemoryCalloc(1, sizeof(SSrvMsg));
srvMsg->msg = tmsg;
srvMsg->type = Release;
srvMsg->pConn = pConn;
tTrace("server conn %p start to release", pConn);
tTrace("server conn %p start to release", exh->handle);
transSendAsync(pThrd->asyncPool, &srvMsg->q);
uvReleaseExHandle(refId);
return;
_return1:
tTrace("server handle %p failed to send to release handle", exh);
uvReleaseExHandle(refId);
return;
_return2:
tTrace("server handle %p failed to send to release handle", exh);
return;
}
void transSendResponse(const STransMsg* pMsg) {
if (pMsg->handle == NULL) {
return;
}
SSrvConn* pConn = pMsg->handle;
SWorkThrdObj* pThrd = pConn->hostThrd;
if (pThrd->quit) {
return;
}
void transSendResponse(const STransMsg* msg) {
SExHandle* exh = msg->handle;
int64_t refId = msg->refId;
ASYNC_CHECK_HANDLE(exh, refId);
assert(refId != 0);
STransMsg tmsg = *msg;
tmsg.refId = refId;
SWorkThrdObj* pThrd = exh->pThrd;
ASYNC_ERR_JRET(pThrd);
SSrvMsg* srvMsg = taosMemoryCalloc(1, sizeof(SSrvMsg));
srvMsg->pConn = pConn;
srvMsg->msg = *pMsg;
srvMsg->msg = tmsg;
srvMsg->type = Normal;
tTrace("server conn %p start to send resp (1/2)", pConn);
tTrace("server conn %p start to send resp (1/2)", exh->handle);
transSendAsync(pThrd->asyncPool, &srvMsg->q);
uvReleaseExHandle(refId);
return;
_return1:
tTrace("server handle %p failed to send resp", exh);
rpcFreeCont(msg->pCont);
uvReleaseExHandle(refId);
return;
_return2:
tTrace("server handle %p failed to send resp", exh);
rpcFreeCont(msg->pCont);
return;
}
void transRegisterMsg(const STransMsg* msg) {
if (msg->handle == NULL) {
return;
}
SSrvConn* pConn = msg->handle;
SWorkThrdObj* pThrd = pConn->hostThrd;
SExHandle* exh = msg->handle;
int64_t refId = msg->refId;
ASYNC_CHECK_HANDLE(exh, refId);
STransMsg tmsg = *msg;
tmsg.refId = refId;
SWorkThrdObj* pThrd = exh->pThrd;
ASYNC_ERR_JRET(pThrd);
SSrvMsg* srvMsg = taosMemoryCalloc(1, sizeof(SSrvMsg));
srvMsg->pConn = pConn;
srvMsg->msg = *msg;
srvMsg->msg = tmsg;
srvMsg->type = Register;
tTrace("server conn %p start to register brokenlink callback", pConn);
tTrace("server conn %p start to register brokenlink callback", exh->handle);
transSendAsync(pThrd->asyncPool, &srvMsg->q);
uvReleaseExHandle(refId);
return;
_return1:
tTrace("server handle %p failed to send to register brokenlink", exh);
rpcFreeCont(msg->pCont);
uvReleaseExHandle(refId);
return;
_return2:
tTrace("server handle %p failed to send to register brokenlink", exh);
rpcFreeCont(msg->pCont);
}
int transGetConnInfo(void* thandle, STransHandleInfo* pInfo) {
SSrvConn* pConn = thandle;
struct sockaddr_in addr = pConn->addr;
int transGetConnInfo(void* thandle, STransHandleInfo* pInfo) {
if (thandle == NULL) {
tTrace("invalid handle %p, failed to Get Conn info", thandle);
return -1;
}
SExHandle* ex = thandle;
SSrvConn* pConn = ex->handle;
struct sockaddr_in addr = pConn->addr;
pInfo->clientIp = (uint32_t)(addr.sin_addr.s_addr);
pInfo->clientPort = ntohs(addr.sin_port);
tstrncpy(pInfo->user, pConn->user, sizeof(pInfo->user));

6
source/os/src/osSystem.c
View File

@ -119,7 +119,7 @@ int taosSetConsoleEcho(bool on) {
#endif
}
void setTerminalMode() {
void taosSetTerminalMode() {
#if defined(WINDOWS)
#else
@ -152,7 +152,7 @@ void setTerminalMode() {
#endif
}
int32_t getOldTerminalMode() {
int32_t taosGetOldTerminalMode() {
#if defined(WINDOWS)
#else
@ -170,7 +170,7 @@ int32_t getOldTerminalMode() {
#endif
}
void resetTerminalMode() {
void taosResetTerminalMode() {
#if defined(WINDOWS)
#else

80
source/util/src/tconfig.c
View File

@ -139,12 +139,6 @@ static int32_t cfgCheckAndSetDir(SConfigItem *pItem, const char *inputDir) {
return -1;
}
if (taosRealPath(fullDir, NULL, PATH_MAX) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
uError("failed to get realpath of dir:%s since %s", inputDir, terrstr());
return -1;
}
taosMemoryFreeClear(pItem->str);
pItem->str = strdup(fullDir);
if (pItem->str == NULL) {
@ -172,9 +166,8 @@ static int32_t cfgSetBool(SConfigItem *pItem, const char *value, ECfgSrcType sty
static int32_t cfgSetInt32(SConfigItem *pItem, const char *value, ECfgSrcType stype) {
int32_t ival = (int32_t)atoi(value);
if (ival < pItem->imin || ival > pItem->imax) {
uError("cfg:%s, type:%s src:%s value:%d out of range[%" PRId64 ", %" PRId64 "], use last src:%s value:%d",
pItem->name, cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), ival, pItem->imin, pItem->imax,
cfgStypeStr(pItem->stype), pItem->i32);
uError("cfg:%s, type:%s src:%s value:%d out of range[%" PRId64 ", %" PRId64 "]", pItem->name,
cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), ival, pItem->imin, pItem->imax);
terrno = TSDB_CODE_OUT_OF_RANGE;
return -1;
}
@ -187,10 +180,8 @@ static int32_t cfgSetInt32(SConfigItem *pItem, const char *value, ECfgSrcType st
static int32_t cfgSetInt64(SConfigItem *pItem, const char *value, ECfgSrcType stype) {
int64_t ival = (int64_t)atoi(value);
if (ival < pItem->imin || ival > pItem->imax) {
uError("cfg:%s, type:%s src:%s value:%" PRId64 " out of range[%" PRId64 ", %" PRId64
"], use last src:%s value:%" PRId64,
pItem->name, cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), ival, pItem->imin, pItem->imax,
cfgStypeStr(pItem->stype), pItem->i64);
uError("cfg:%s, type:%s src:%s value:%" PRId64 " out of range[%" PRId64 ", %" PRId64 "]", pItem->name,
cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), ival, pItem->imin, pItem->imax);
terrno = TSDB_CODE_OUT_OF_RANGE;
return -1;
}
@ -203,9 +194,8 @@ static int32_t cfgSetInt64(SConfigItem *pItem, const char *value, ECfgSrcType st
static int32_t cfgSetFloat(SConfigItem *pItem, const char *value, ECfgSrcType stype) {
float fval = (float)atof(value);
if (fval < pItem->fmin || fval > pItem->fmax) {
uError("cfg:%s, type:%s src:%s value:%f out of range[%f, %f], use last src:%s value:%f", pItem->name,
cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), fval, pItem->fmin, pItem->fmax, cfgStypeStr(pItem->stype),
pItem->fval);
uError("cfg:%s, type:%s src:%s value:%f out of range[%f, %f]", pItem->name, cfgDtypeStr(pItem->dtype),
cfgStypeStr(stype), fval, pItem->fmin, pItem->fmax);
terrno = TSDB_CODE_OUT_OF_RANGE;
return -1;
}
@ -219,8 +209,8 @@ static int32_t cfgSetString(SConfigItem *pItem, const char *value, ECfgSrcType s
char *tmp = strdup(value);
if (tmp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name,
cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str);
uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s", pItem->name, cfgDtypeStr(pItem->dtype),
cfgStypeStr(stype), value, terrstr());
return -1;
}
@ -232,9 +222,8 @@ static int32_t cfgSetString(SConfigItem *pItem, const char *value, ECfgSrcType s
static int32_t cfgSetDir(SConfigItem *pItem, const char *value, ECfgSrcType stype) {
if (cfgCheckAndSetDir(pItem, value) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name,
cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str);
uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s", pItem->name, cfgDtypeStr(pItem->dtype),
cfgStypeStr(stype), value, terrstr());
return -1;
}
@ -245,8 +234,8 @@ static int32_t cfgSetDir(SConfigItem *pItem, const char *value, ECfgSrcType styp
static int32_t cfgSetLocale(SConfigItem *pItem, const char *value, ECfgSrcType stype) {
if (cfgCheckAndSetLocale(pItem, value) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name,
cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str);
uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s", pItem->name, cfgDtypeStr(pItem->dtype),
cfgStypeStr(stype), value, terrstr());
return -1;
}
@ -257,8 +246,8 @@ static int32_t cfgSetLocale(SConfigItem *pItem, const char *value, ECfgSrcType s
static int32_t cfgSetCharset(SConfigItem *pItem, const char *value, ECfgSrcType stype) {
if (cfgCheckAndSetCharset(pItem, value) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name,
cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str);
uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s", pItem->name, cfgDtypeStr(pItem->dtype),
cfgStypeStr(stype), value, terrstr());
return -1;
}
@ -269,8 +258,8 @@ static int32_t cfgSetCharset(SConfigItem *pItem, const char *value, ECfgSrcType
static int32_t cfgSetTimezone(SConfigItem *pItem, const char *value, ECfgSrcType stype) {
if (cfgCheckAndSetTimezone(pItem, value) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name,
cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str);
uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s", pItem->name, cfgDtypeStr(pItem->dtype),
cfgStypeStr(stype), value, terrstr());
return -1;
}
@ -524,6 +513,8 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) {
for (int32_t i = 0; i < size; ++i) {
SConfigItem *pItem = taosArrayGet(pCfg->array, i);
if (tsc && !pItem->tsc) continue;
if (dump && strcmp(pItem->name, "scriptDir") == 0) continue;
if (dump && strcmp(pItem->name, "simDebugFlag") == 0) continue;
tstrncpy(src, cfgStypeStr(pItem->stype), CFG_SRC_PRINT_LEN);
for (int32_t i = 0; i < CFG_SRC_PRINT_LEN; ++i) {
if (src[i] == 0) src[i] = ' ';
@ -562,10 +553,10 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) {
break;
case CFG_DTYPE_FLOAT:
if (dump) {
printf("%s %s %f", src, name, pItem->fval);
printf("%s %s %.2f", src, name, pItem->fval);
printf("\n");
} else {
uInfo("%s %s %f", src, name, pItem->fval);
uInfo("%s %s %.2f", src, name, pItem->fval);
}
break;
case CFG_DTYPE_STRING:
@ -606,15 +597,19 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) {
char *line = NULL, *name, *value, *value2, *value3;
int32_t olen, vlen, vlen2, vlen3;
ssize_t _bytes = 0;
if (taosIsDir(filepath)) {
return -1;
}
int32_t code = 0;
TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ | TD_FILE_STREAM);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
// success when the file does not exist
if (errno == ENOENT) {
terrno = TAOS_SYSTEM_ERROR(errno);
uInfo("failed to load from cfg file %s since %s, use default parameters", filepath, terrstr());
return 0;
} else {
uError("failed to load from cfg file %s since %s", filepath, terrstr());
return -1;
}
}
while (!taosEOFFile(pFile)) {
@ -643,17 +638,24 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) {
if (vlen3 != 0) value3[vlen3] = 0;
}
cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE);
code = cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE);
if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) {
cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_CFG_FILE);
code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_CFG_FILE);
if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
}
}
taosCloseFile(&pFile);
if (line != NULL) taosMemoryFreeClear(line);
uInfo("load from cfg file %s success", filepath);
return 0;
if (code == 0 || (code != 0 && terrno == TSDB_CODE_CFG_NOT_FOUND)) {
uInfo("load from cfg file %s success", filepath);
return 0;
} else {
uError("failed to load from cfg file %s since %s", filepath, terrstr());
return -1;
}
}
int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) {

31
source/util/src/tlog.c
View File

@ -137,24 +137,23 @@ static void taosStopLog() {
}
}
static void taosLogBuffDestroy() {
taosThreadMutexDestroy(&tsLogObj.logHandle->buffMutex);
taosCloseFile(&tsLogObj.logHandle->pFile);
taosMemoryFreeClear(tsLogObj.logHandle->buffer);
memset(&tsLogObj.logHandle->buffer, 0, sizeof(tsLogObj.logHandle->buffer));
taosThreadMutexDestroy(&tsLogObj.logMutex);
taosMemoryFreeClear(tsLogObj.logHandle);
memset(&tsLogObj.logHandle, 0, sizeof(tsLogObj.logHandle));
tsLogObj.logHandle = NULL;
}
void taosCloseLog() {
taosStopLog();
if (taosCheckPthreadValid(tsLogObj.logHandle->asyncThread)) {
taosThreadJoin(tsLogObj.logHandle->asyncThread, NULL);
if (tsLogObj.logHandle != NULL) {
taosStopLog();
if (tsLogObj.logHandle != NULL && taosCheckPthreadValid(tsLogObj.logHandle->asyncThread)) {
taosThreadJoin(tsLogObj.logHandle->asyncThread, NULL);
}
tsLogInited = 0;
taosThreadMutexDestroy(&tsLogObj.logHandle->buffMutex);
taosCloseFile(&tsLogObj.logHandle->pFile);
taosMemoryFreeClear(tsLogObj.logHandle->buffer);
memset(&tsLogObj.logHandle->buffer, 0, sizeof(tsLogObj.logHandle->buffer));
taosThreadMutexDestroy(&tsLogObj.logMutex);
taosMemoryFreeClear(tsLogObj.logHandle);
memset(&tsLogObj.logHandle, 0, sizeof(tsLogObj.logHandle));
tsLogObj.logHandle = NULL;
}
tsLogInited = 0;
taosLogBuffDestroy(tsLogObj.logHandle);
}
static bool taosLockLogFile(TdFilePtr pFile) {

23
source/util/src/tprocess.c
View File

@ -154,7 +154,8 @@ static void taosProcCleanupQueue(SProcQueue *pQueue) {
}
static int32_t taosProcQueuePush(SProcObj *pProc, SProcQueue *pQueue, const char *pHead, int16_t rawHeadLen,
const char *pBody, int32_t rawBodyLen, int64_t handle, EProcFuncType ftype) {
const char *pBody, int32_t rawBodyLen, int64_t handle, int64_t handleRef,
EProcFuncType ftype) {
if (rawHeadLen == 0 || pHead == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return -1;
@ -172,7 +173,7 @@ static int32_t taosProcQueuePush(SProcObj *pProc, SProcQueue *pQueue, const char
}
if (handle != 0 && ftype == PROC_FUNC_REQ) {
if (taosHashPut(pProc->hash, &handle, sizeof(int64_t), &handle, sizeof(int64_t)) != 0) {
if (taosHashPut(pProc->hash, &handle, sizeof(int64_t), &handleRef, sizeof(int64_t)) != 0) {
taosThreadMutexUnlock(&pQueue->mutex);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
@ -286,13 +287,13 @@ static int32_t taosProcQueuePop(SProcQueue *pQueue, void **ppHead, int16_t *pHea
pQueue->head = headLen + bodyLen;
} else if (remain < 8 + headLen) {
memcpy(pHead, pQueue->pBuffer + pQueue->head + 8, remain - 8);
memcpy((char*)pHead + remain - 8, pQueue->pBuffer, headLen - (remain - 8));
memcpy((char *)pHead + remain - 8, pQueue->pBuffer, headLen - (remain - 8));
memcpy(pBody, pQueue->pBuffer + headLen - (remain - 8), bodyLen);
pQueue->head = headLen - (remain - 8) + bodyLen;
} else if (remain < 8 + headLen + bodyLen) {
memcpy(pHead, pQueue->pBuffer + pQueue->head + 8, headLen);
memcpy(pBody, pQueue->pBuffer + pQueue->head + 8 + headLen, remain - 8 - headLen);
memcpy((char*)pBody + remain - 8 - headLen, pQueue->pBuffer, bodyLen - (remain - 8 - headLen));
memcpy((char *)pBody + remain - 8 - headLen, pQueue->pBuffer, bodyLen - (remain - 8 - headLen));
pQueue->head = bodyLen - (remain - 8 - headLen);
} else {
memcpy(pHead, pQueue->pBuffer + pQueue->head + 8, headLen);
@ -454,19 +455,25 @@ void taosProcCleanup(SProcObj *pProc) {
}
int32_t taosProcPutToChildQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
void *handle, EProcFuncType ftype) {
void *handle, int64_t handleRef, EProcFuncType ftype) {
if (ftype != PROC_FUNC_REQ) {
terrno = TSDB_CODE_INVALID_PARA;
return -1;
}
return taosProcQueuePush(pProc, pProc->pChildQueue, pHead, headLen, pBody, bodyLen, (int64_t)handle, ftype);
return taosProcQueuePush(pProc, pProc->pChildQueue, pHead, headLen, pBody, bodyLen, (int64_t)handle, handleRef,
ftype);
}
void taosProcRemoveHandle(SProcObj *pProc, void *handle) {
int64_t taosProcRemoveHandle(SProcObj *pProc, void *handle) {
int64_t h = (int64_t)handle;
taosThreadMutexLock(&pProc->pChildQueue->mutex);
int64_t *handleRef = taosHashGet(pProc->hash, &h, sizeof(int64_t));
taosHashRemove(pProc->hash, &h, sizeof(int64_t));
taosThreadMutexUnlock(&pProc->pChildQueue->mutex);
if (handleRef == NULL) return 0;
return *handleRef;
}
void taosProcCloseHandles(SProcObj *pProc, void (*HandleFp)(void *handle)) {
@ -484,7 +491,7 @@ void taosProcCloseHandles(SProcObj *pProc, void (*HandleFp)(void *handle)) {
void taosProcPutToParentQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
EProcFuncType ftype) {
int32_t retry = 0;
while (taosProcQueuePush(pProc, pProc->pParentQueue, pHead, headLen, pBody, bodyLen, 0, ftype) != 0) {
while (taosProcQueuePush(pProc, pProc->pParentQueue, pHead, headLen, pBody, bodyLen, 0, 0, ftype) != 0) {
uWarn("proc:%s, failed to put to queue:%p since %s, retry:%d", pProc->name, pProc->pParentQueue, terrstr(), retry);
retry++;
taosMsleep(retry);

2
source/util/src/ttimer.c
View File

@ -110,7 +110,7 @@ typedef struct time_wheel_t {
tmr_obj_t** slots;
} time_wheel_t;
int32_t tsMaxTmrCtrl = 512;
static int32_t tsMaxTmrCtrl = 512;
static TdThreadOnce tmrModuleInit = PTHREAD_ONCE_INIT;
static TdThreadMutex tmrCtrlMutex;

31
source/util/test/procTest.cpp
View File

@ -120,20 +120,20 @@ TEST_F(UtilTesProc, 01_Push_Pop_Child) {
SProcObj *cproc = taosProcInit(&cfg);
ASSERT_NE(cproc, nullptr);
ASSERT_NE(taosProcPutToChildQ(cproc, &head, 0, body, 0, 0, PROC_FUNC_RSP), 0);
ASSERT_NE(taosProcPutToChildQ(cproc, &head, 0, body, 0, 0, PROC_FUNC_REGIST), 0);
ASSERT_NE(taosProcPutToChildQ(cproc, &head, 0, body, 0, 0, PROC_FUNC_RELEASE), 0);
ASSERT_NE(taosProcPutToChildQ(cproc, NULL, 12, body, 0, 0, PROC_FUNC_REQ), 0);
ASSERT_NE(taosProcPutToChildQ(cproc, &head, 0, body, 0, 0, PROC_FUNC_REQ), 0);
ASSERT_NE(taosProcPutToChildQ(cproc, &head, shm.size, body, 0, 0, PROC_FUNC_REQ), 0);
ASSERT_NE(taosProcPutToChildQ(cproc, &head, sizeof(STestMsg), body, shm.size, 0, PROC_FUNC_REQ), 0);
ASSERT_NE(taosProcPutToChildQ(cproc, &head, 0, body, 0, 0, 0, PROC_FUNC_RSP), 0);
ASSERT_NE(taosProcPutToChildQ(cproc, &head, 0, body, 0, 0, 0, PROC_FUNC_REGIST), 0);
ASSERT_NE(taosProcPutToChildQ(cproc, &head, 0, body, 0, 0, 0, PROC_FUNC_RELEASE), 0);
ASSERT_NE(taosProcPutToChildQ(cproc, NULL, 12, body, 0, 0, 0, PROC_FUNC_REQ), 0);
ASSERT_NE(taosProcPutToChildQ(cproc, &head, 0, body, 0, 0, 0, PROC_FUNC_REQ), 0);
ASSERT_NE(taosProcPutToChildQ(cproc, &head, shm.size, body, 0, 0, 0, PROC_FUNC_REQ), 0);
ASSERT_NE(taosProcPutToChildQ(cproc, &head, sizeof(STestMsg), body, shm.size, 0, 0, PROC_FUNC_REQ), 0);
for (int32_t j = 0; j < 1000; j++) {
int32_t i = 0;
for (i = 0; i < 20; ++i) {
ASSERT_EQ(taosProcPutToChildQ(cproc, &head, sizeof(STestMsg), body, i, 0, PROC_FUNC_REQ), 0);
ASSERT_EQ(taosProcPutToChildQ(cproc, &head, sizeof(STestMsg), body, i, 0, 0, PROC_FUNC_REQ), 0);
}
ASSERT_NE(taosProcPutToChildQ(cproc, &head, sizeof(STestMsg), body, i, 0, PROC_FUNC_REQ), 0);
ASSERT_NE(taosProcPutToChildQ(cproc, &head, sizeof(STestMsg), body, i, 0, 0, PROC_FUNC_REQ), 0);
cfg.isChild = true;
cfg.name = "1235_p";
@ -236,7 +236,7 @@ TEST_F(UtilTesProc, 03_Handle) {
int32_t i = 0;
for (i = 0; i < 20; ++i) {
head.handle = (void *)((int64_t)i);
ASSERT_EQ(taosProcPutToChildQ(cproc, &head, sizeof(STestMsg), body, i, (void *)((int64_t)i), PROC_FUNC_REQ), 0);
ASSERT_EQ(taosProcPutToChildQ(cproc, &head, sizeof(STestMsg), body, i, (void *)((int64_t)i), i, PROC_FUNC_REQ), 0);
}
cfg.isChild = true;
@ -246,9 +246,14 @@ TEST_F(UtilTesProc, 03_Handle) {
taosProcRun(pproc);
taosProcCleanup(pproc);
taosProcRemoveHandle(cproc, (void *)((int64_t)3));
taosProcRemoveHandle(cproc, (void *)((int64_t)5));
taosProcRemoveHandle(cproc, (void *)((int64_t)6));
int64_t ref = 0;
ref = taosProcRemoveHandle(cproc, (void *)((int64_t)3));
EXPECT_EQ(ref, 3);
ref = taosProcRemoveHandle(cproc, (void *)((int64_t)5));
EXPECT_EQ(ref, 5);
ref = taosProcRemoveHandle(cproc, (void *)((int64_t)6));
EXPECT_EQ(ref, 6);
taosProcCloseHandles(cproc, processHandle);
}

4
tests/script/jenkins/basic.txt
View File

@ -76,7 +76,7 @@
./test.sh -f tsim/insert/backquote.sim -m
./test.sh -f tsim/parser/fourArithmetic-basic.sim -m
./test.sh -f tsim/query/interval-offset.sim -m
./test.sh -f tsim/tmq/basic1.sim -m
#./test.sh -f tsim/tmq/basic1.sim -m
./test.sh -f tsim/stable/vnode3.sim -m
./test.sh -f tsim/qnode/basic1.sim -m
./test.sh -f tsim/mnode/basic1.sim -m
@ -85,6 +85,6 @@
./test.sh -f tsim/sma/tsmaCreateInsertData.sim
# --- valgrind
./test.sh -f tsim/valgrind/checkError.sim -v
#./test.sh -f tsim/valgrind/checkError.sim -v
#======================b1-end===============

103
tests/script/tsim/query/crash_sql.sim Normal file
View File

@ -0,0 +1,103 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
$loop_cnt = 0
check_dnode_ready:
$loop_cnt = $loop_cnt + 1
sleep 200
if $loop_cnt == 10 then
print ====> dnode not ready!
return -1
endi
sql show dnodes
print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
if $data00 != 1 then
return -1
endi
if $data04 != ready then
goto check_dnode_ready
endi
sql connect
print =============== create database
sql create database db
sql show databases
if $rows != 2 then
return -1
endi
sql use db
print =============== create super table and child table
sql create table stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int)
sql show stables
print $rows $data00 $data01 $data02
if $rows != 1 then
return -1
endi
sql create table ct1 using stb1 tags ( 1 )
sql create table ct2 using stb1 tags ( 2 )
sql create table ct3 using stb1 tags ( 3 )
sql create table ct4 using stb1 tags ( 4 )
sql show tables
print $rows $data00 $data10 $data20
if $rows != 4 then
return -1
endi
print =============== insert data into child table ct1 (s)
sql insert into ct1 values ( '2022-01-01 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now+1a )
sql insert into ct1 values ( '2022-01-01 01:01:06.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now+2a )
sql insert into ct1 values ( '2022-01-01 01:01:10.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now+3a )
sql insert into ct1 values ( '2022-01-01 01:01:16.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now+4a )
sql insert into ct1 values ( '2022-01-01 01:01:20.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now+5a )
sql insert into ct1 values ( '2022-01-01 01:01:26.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now+6a )
sql insert into ct1 values ( '2022-01-01 01:01:30.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", now+7a )
sql insert into ct1 values ( '2022-01-01 01:01:36.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", now+8a )
print =============== insert data into child table ct2 (d)
sql insert into ct2 values ( '2022-01-01 01:00:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now+1a )
sql insert into ct2 values ( '2022-01-01 10:00:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now+2a )
sql insert into ct2 values ( '2022-01-01 20:00:01.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now+3a )
sql insert into ct2 values ( '2022-01-02 10:00:01.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now+4a )
sql insert into ct2 values ( '2022-01-02 20:00:01.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now+5a )
sql insert into ct2 values ( '2022-01-03 10:00:01.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", now+6a )
sql insert into ct2 values ( '2022-01-03 20:00:01.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", now+7a )
print =============== insert data into child table ct3 (n)
sql insert into ct3 values ( '2021-12-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
sql insert into ct3 values ( '2021-12-31 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now+1a )
sql insert into ct3 values ( '2022-01-01 01:01:06.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now+2a )
sql insert into ct3 values ( '2022-01-07 01:01:10.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now+3a )
sql insert into ct3 values ( '2022-01-31 01:01:16.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now+4a )
sql insert into ct3 values ( '2022-02-01 01:01:20.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now+5a )
sql insert into ct3 values ( '2022-02-28 01:01:26.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now+6a )
sql insert into ct3 values ( '2022-03-01 01:01:30.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" )
sql insert into ct3 values ( '2022-03-08 01:01:36.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" )
print =============== insert data into child table ct4 (y)
sql insert into ct4 values ( '2019-01-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
sql insert into ct4 values ( '2019-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now+1a )
sql insert into ct4 values ( '2019-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now+2a )
sql insert into ct4 values ( '2020-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now+3a )
sql insert into ct4 values ( '2020-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now+4a )
sql insert into ct4 values ( '2020-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now+5a )
sql insert into ct4 values ( '2020-12-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
sql insert into ct4 values ( '2021-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now+6a )
sql insert into ct4 values ( '2021-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" )
sql insert into ct4 values ( '2021-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" )
sql insert into ct4 values ( '2022-02-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" )
sql insert into ct4 values ( '2022-05-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
print ================ start query ======================
print ================ SQL used to cause taosd or taos shell crash
sql select sum(c1) ,count(c1) from ct4 group by c1 having sum(c10) between 0 and 1 ;
#system sh/exec.sh -n dnode1 -s stop -x SIGINT

4
tests/script/tsim/tmq/basic1.sim
View File

@ -175,7 +175,7 @@ $totalMsgOfCtb = $rowsPerCtb
$expectmsgcnt = $totalMsgOfCtb
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
print == start consumer to pull msgs from stb
print == start consumer to pull msgs from ctb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
@ -246,7 +246,7 @@ $totalMsgOfNtb = $rowsPerCtb
$expectmsgcnt = $totalMsgOfNtb
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
print == start consumer to pull msgs from stb
print == start consumer to pull msgs from ntb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start

134
tests/script/tsim/tmq/basic1Of2Cons.sim
View File

@ -132,39 +132,32 @@ if $data[0][1] == 1 then
endi
endi
if $data[0][2] != 0 then
if $data[0][2] != $expectmsgcnt then
return -1
# either $data[0][2] == $totalMsgOfStb and $data[1][2] == 0
# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfStb
if $data[0][2] == $totalMsgOfStb then
if $data[1][2] == 0 then
goto check_ok_0
endi
if $data[1][2] != 0 then
return -1
endi
endi
if $data[1][2] != 0 then
if $data[1][2] != $expectmsgcnt then
return -1
endi
if $data[0][2] != 0 then
return -1
elif $data[0][2] == 0 then
if $data[1][2] == $totalMsgOfStb then
goto check_ok_0
endi
endi
return -1
check_ok_0:
if $data[0][3] != 0 then
if $data[0][3] != $expectmsgcnt then
return -1
if $data[0][3] == $totalMsgOfStb then
if $data[1][3] == 0 then
goto check_ok_1
endi
if $data[1][3] != 0 then
return -1
endi
endi
if $data[1][3] != 0 then
if $data[1][3] != $expectmsgcnt then
return -1
endi
if $data[0][3] != 0 then
return -1
elif $data[0][3] == 0 then
if $data[1][3] == $totalMsgOfStb then
goto check_ok_1
endi
endi
return -1
check_ok_1:
$loop_cnt = $loop_cnt + 1
goto loop_consume_diff_topic_from_stb
loop_consume_diff_topic_from_stb_end:
@ -241,39 +234,31 @@ if $data[0][1] == 1 then
endi
endi
if $data[0][2] != $totalMsgOfCtb then
if $data[1][2] != $totalMsgOfCtb then
return -1
# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0
# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb
if $data[0][2] == $totalMsgOfCtb then
if $data[1][2] == 0 then
goto check_ok_2
endi
if $data[0][2] != 0 then
return -1
endi
endi
if $data[1][2] != $totalMsgOfCtb then
if $data[0][2] != $totalMsgOfCtb then
return -1
endi
if $data[1][2] != 0 then
return -1
elif $data[0][2] == 0 then
if $data[1][2] == $totalMsgOfCtb then
goto check_ok_2
endi
endi
return -1
check_ok_2:
if $data[0][3] != $totalMsgOfCtb then
if $data[1][3] != $totalMsgOfCtb then
return -1
if $data[0][3] == $totalMsgOfCtb then
if $data[1][3] == 0 then
goto check_ok_3
endi
if $data[0][3] != 0 then
return -1
endi
endi
if $data[1][3] != $totalMsgOfCtb then
if $data[0][3] != $totalMsgOfCtb then
return -1
elif $data[0][3] == 0 then
if $data[1][3] == $totalMsgOfCtb then
goto check_ok_3
endi
if $data[1][3] != 0 then
return -1
endi
endi
return -1
check_ok_3:
$loop_cnt = $loop_cnt + 1
goto loop_consume_diff_topic_from_ctb
@ -351,39 +336,32 @@ if $data[1][1] == 0 then
endi
endi
if $data[0][2] != $totalMsgOfNtb then
if $data[1][2] != $totalMsgOfNtb then
return -1
# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0
# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb
if $data[0][2] == $totalMsgOfNtb then
if $data[1][2] == 0 then
goto check_ok_4
endi
if $data[0][2] != 0 then
return -1
endi
endi
if $data[1][2] != $totalMsgOfNtb then
if $data[0][2] != $totalMsgOfNtb then
return -1
endi
if $data[1][2] != 0 then
return -1
elif $data[0][2] == 0 then
if $data[1][2] == $totalMsgOfNtb then
goto check_ok_4
endi
endi
return -1
check_ok_4:
if $data[0][3] != $totalMsgOfNtb then
if $data[1][3] != $totalMsgOfNtb then
return -1
if $data[0][3] == $totalMsgOfNtb then
if $data[1][3] == 0 then
goto check_ok_5
endi
if $data[0][3] != 0 then
return -1
endi
endi
if $data[1][3] != $totalMsgOfNtb then
if $data[0][3] != $totalMsgOfNtb then
return -1
elif $data[0][3] == 0 then
if $data[1][3] == $totalMsgOfNtb then
goto check_ok_5
endi
if $data[1][3] != 0 then
return -1
endi
endi
return -1
check_ok_5:
$loop_cnt = $loop_cnt + 1
goto loop_consume_diff_topic_from_ntb
loop_consume_diff_topic_from_ntb_end:

4
tests/script/tsim/tmq/basic2.sim
View File

@ -133,7 +133,7 @@ $totalMsgOfCtb = $rowsPerCtb * $topicNum
$expectmsgcnt = $totalMsgOfCtb
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
print == start consumer to pull msgs from stb
print == start consumer to pull msgs from ctb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
@ -190,7 +190,7 @@ $totalMsgOfNtb = $rowsPerCtb * $topicNum
$expectmsgcnt = $totalMsgOfNtb
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
print == start consumer to pull msgs from stb
print == start consumer to pull msgs from ntb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start

132
tests/script/tsim/tmq/basic2Of2Cons.sim
View File

@ -103,39 +103,31 @@ if $data[0][1] == 1 then
endi
endi
if $data[0][2] != 0 then
if $data[0][2] != $expectmsgcnt then
return -1
# either $data[0][2] == $totalMsgOfStb and $data[1][2] == 0
# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfStb
if $data[0][2] == $totalMsgOfStb then
if $data[1][2] == 0 then
goto check_ok_0
endi
if $data[1][2] != 0 then
return -1
endi
endi
if $data[1][2] != 0 then
if $data[1][2] != $expectmsgcnt then
return -1
endi
if $data[0][2] != 0 then
return -1
elif $data[0][2] == 0 then
if $data[1][2] == $totalMsgOfStb then
goto check_ok_0
endi
endi
return -1
check_ok_0:
if $data[0][3] != 0 then
if $data[0][3] != $expectmsgcnt then
return -1
if $data[0][3] == $totalMsgOfStb then
if $data[1][3] == 0 then
goto check_ok_1
endi
if $data[1][3] != 0 then
return -1
endi
endi
if $data[1][3] != 0 then
if $data[1][3] != $expectmsgcnt then
return -1
endi
if $data[0][3] != 0 then
return -1
elif $data[0][3] == 0 then
if $data[1][3] == $totalMsgOfStb then
goto check_ok_1
endi
endi
return -1
check_ok_1:
#######################################################################################
# clear consume info and consume result
@ -198,39 +190,31 @@ if $data[0][1] == 1 then
endi
endi
if $data[0][2] != $totalMsgOfCtb then
if $data[1][2] != $totalMsgOfCtb then
return -1
# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0
# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb
if $data[0][2] == $totalMsgOfCtb then
if $data[1][2] == 0 then
goto check_ok_2
endi
if $data[0][2] != 0 then
return -1
endi
endi
if $data[1][2] != $totalMsgOfCtb then
if $data[0][2] != $totalMsgOfCtb then
return -1
endi
if $data[1][2] != 0 then
return -1
elif $data[0][2] == 0 then
if $data[1][2] == $totalMsgOfCtb then
goto check_ok_2
endi
endi
return -1
check_ok_2:
if $data[0][3] != $totalMsgOfCtb then
if $data[1][3] != $totalMsgOfCtb then
return -1
if $data[0][3] == $totalMsgOfCtb then
if $data[1][3] == 0 then
goto check_ok_3
endi
if $data[0][3] != 0 then
return -1
endi
endi
if $data[1][3] != $totalMsgOfCtb then
if $data[0][3] != $totalMsgOfCtb then
return -1
elif $data[0][3] == 0 then
if $data[1][3] == $totalMsgOfCtb then
goto check_ok_3
endi
if $data[1][3] != 0 then
return -1
endi
endi
return -1
check_ok_3:
#######################################################################################
# clear consume info and consume result
@ -293,39 +277,31 @@ if $data[1][1] == 0 then
endi
endi
if $data[0][2] != $totalMsgOfNtb then
if $data[1][2] != $totalMsgOfNtb then
return -1
# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0
# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb
if $data[0][2] == $totalMsgOfNtb then
if $data[1][2] == 0 then
goto check_ok_4
endi
if $data[0][2] != 0 then
return -1
endi
endi
if $data[1][2] != $totalMsgOfNtb then
if $data[0][2] != $totalMsgOfNtb then
return -1
endi
if $data[1][2] != 0 then
return -1
elif $data[0][2] == 0 then
if $data[1][2] == $totalMsgOfNtb then
goto check_ok_4
endi
endi
return -1
check_ok_4:
if $data[0][3] != $totalMsgOfNtb then
if $data[1][3] != $totalMsgOfNtb then
return -1
if $data[0][3] == $totalMsgOfNtb then
if $data[1][3] == 0 then
goto check_ok_5
endi
if $data[0][3] != 0 then
return -1
endi
endi
if $data[1][3] != $totalMsgOfNtb then
if $data[0][3] != $totalMsgOfNtb then
return -1
elif $data[0][3] == 0 then
if $data[1][3] == $totalMsgOfNtb then
goto check_ok_5
endi
if $data[1][3] != 0 then
return -1
endi
endi
return -1
check_ok_5:
#------ not need stop consumer, because it exit after pull msg overthan expect msg
#system tsim/tmq/consume.sh -s stop -x SIGINT

337
tests/script/tsim/tmq/basic2Of2ConsOverlap.sim Normal file
View File

@ -0,0 +1,337 @@
#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
#basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics
#basic2Of2ConsOverlap.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics
#basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics
#basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics
# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
#
# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
#
run tsim/tmq/prepareBasicEnv-1vgrp.sim
#---- global parameters start ----#
$dbName = db
$vgroups = 1
$stbPrefix = stb
$ctbPrefix = ctb
$ntbPrefix = ntb
$stbNum = 1
$ctbNum = 10
$ntbNum = 10
$rowsPerCtb = 10
$tstart = 1640966400000 # 2022-01-01 00:00:00.000
#---- global parameters end ----#
$pullDelay = 5
$ifcheckdata = 1
$showMsg = 1
$showRow = 0
sql connect
sql use $dbName
print == create topics from super table
sql create topic topic_stb_column as select ts, c3 from stb
sql create topic topic_stb_all as select ts, c1, c2, c3 from stb
sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
print == create topics from child table
sql create topic topic_ctb_column as select ts, c3 from ctb0
sql create topic topic_ctb_all as select * from ctb0
sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0
print == create topics from normal table
sql create topic topic_ntb_column as select ts, c3 from ntb0
sql create topic topic_ntb_all as select * from ntb0
sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0
#sql show topics
#if $rows != 9 then
# return -1
#endi
$keyList = ' . group.id:cgrp1
$keyList = $keyList . '
$topicNum = 2
#=============================== start consume =============================#
print ================ test consume from stb
print == overlap toipcs: topic_stb_column + topic_stb_all, topic_stb_function + topic_stb_all
$topicList = ' . topic_stb_column
$topicList = $topicList . ,
$topicList = $topicList . topic_stb_all
$topicList = $topicList . '
$consumerId = 0
$totalMsgOfOneTopic = $ctbNum * $rowsPerCtb
$totalMsgOfStb = $totalMsgOfOneTopic * $topicNum
$expectmsgcnt = $totalMsgOfStb
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
$topicList = ' . topic_stb_all
$topicList = $topicList . ,
$topicList = $topicList . topic_stb_function
$topicList = $topicList . '
$consumerId = 1
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
print == start consumer to pull msgs from stb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
print == check consume result
wait_consumer_end_from_stb:
sql select * from consumeresult
print ==> rows: $rows
print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
if $rows != 2 then
sleep 1000
goto wait_consumer_end_from_stb
endi
if $data[0][1] == 0 then
if $data[1][1] != 1 then
return -1
endi
endi
if $data[0][1] == 1 then
if $data[1][1] != 0 then
return -1
endi
endi
# $data[0][2]/$data[1][2] should be between $totalMsgOfOneTopic and $totalMsgOfStb.
if $data[0][2] < $totalMsgOfOneTopic then
return -1
endi
if $data[0][2] > $totalMsgOfStb then
return -1
endi
if $data[1][2] < $totalMsgOfOneTopic then
return -1
endi
if $data[1][2] > $totalMsgOfStb then
return -1
endi
$totalMsgCons = $totalMsgOfOneTopic + $totalMsgOfStb
$sumOfMsgCnt = $data[0][2] + $data[1][2]
if $sumOfMsgCnt != $totalMsgCons then
return -1
endi
# $data[0][3]/$data[1][3] should be between $totalMsgOfOneTopic and $totalMsgOfStb.
if $data[0][3] < $totalMsgOfOneTopic then
return -1
endi
if $data[0][3] > $totalMsgOfStb then
return -1
endi
if $data[1][3] < $totalMsgOfOneTopic then
return -1
endi
if $data[1][3] > $totalMsgOfStb then
return -1
endi
$totalMsgCons = $totalMsgOfOneTopic + $totalMsgOfStb
$sumOfRows = $data[0][3] + $data[1][3]
if $sumOfRows != $totalMsgCons then
return -1
endi
#######################################################################################
# clear consume info and consume result
#run tsim/tmq/clearConsume.sim
# because drop table function no stable, so by create new db for consume info and result. Modify it later
$cdbName = cdb1
sql create database $cdbName vgroups 1
sleep 500
sql use $cdbName
print == create consume info table and consume result table for ctb
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)
sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
sql show tables
if $rows != 2 then
return -1
endi
#######################################################################################
print ================ test consume from ctb
print == overlap toipcs: topic_ctb_column + topic_ctb_all, topic_ctb_function + topic_ctb_all
$topicList = ' . topic_ctb_column
$topicList = $topicList . ,
$topicList = $topicList . topic_ctb_all
$topicList = $topicList . '
$consumerId = 0
$totalMsgOfOneTopic = $rowsPerCtb
$totalMsgOfCtb = $totalMsgOfOneTopic * $topicNum
$expectmsgcnt = $totalMsgOfCtb
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
$topicList = ' . topic_ctb_function
$topicList = $topicList . ,
$topicList = $topicList . topic_ctb_all
$topicList = $topicList . '
$consumerId = 1
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
print == start consumer to pull msgs from ctb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
print == check consume result
wait_consumer_end_from_ctb:
sql select * from consumeresult
print ==> rows: $rows
print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
if $rows != 2 then
sleep 1000
goto wait_consumer_end_from_ctb
endi
if $data[0][1] == 0 then
if $data[1][1] != 1 then
return -1
endi
endi
if $data[0][1] == 1 then
if $data[1][1] != 0 then
return -1
endi
endi
# either $data[0][2] $totalMsgOfOneTopic and $data[1][2] == $totalMsgOfCtb
# or $data[0][2] $totalMsgOfCtb and $data[1][2] == $totalMsgOfOneTopic
if $data[0][2] == $totalMsgOfOneTopic then
if $data[1][2] == $totalMsgOfCtb then
goto check_ok_0
endi
elif $data[1][2] == $totalMsgOfOneTopic then
if $data[0][2] == $totalMsgOfCtb then
goto check_ok_0
endi
endi
return -1
check_ok_0:
if $data[0][3] == $totalMsgOfOneTopic then
if $data[1][3] == $totalMsgOfCtb then
goto check_ok_1
endi
elif $data[1][3] == $totalMsgOfOneTopic then
if $data[0][3] == $totalMsgOfCtb then
goto check_ok_1
endi
endi
return -1
check_ok_1:
#######################################################################################
# clear consume info and consume result
#run tsim/tmq/clearConsume.sim
# because drop table function no stable, so by create new db for consume info and result. Modify it later
$cdbName = cdb2
sql create database $cdbName vgroups 1
sleep 500
sql use $cdbName
print == create consume info table and consume result table for ntb
sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)
sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
sql show tables
if $rows != 2 then
return -1
endi
#######################################################################################
print ================ test consume from ntb
print == overlap toipcs: topic_ntb_column + topic_ntb_all, topic_ntb_function + topic_ntb_all
$topicList = ' . topic_ntb_column
$topicList = $topicList . ,
$topicList = $topicList . topic_ntb_all
$topicList = $topicList . '
$consumerId = 0
$totalMsgOfOneTopic = $rowsPerCtb
$totalMsgOfNtb = $totalMsgOfOneTopic * $topicNum
$expectmsgcnt = $totalMsgOfNtb
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
$topicList = ' . topic_ntb_function
$topicList = $topicList . ,
$topicList = $topicList . topic_ntb_all
$topicList = $topicList . '
$consumerId = 1
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
print == start consumer to pull msgs from ntb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
print == check consume result from ntb
wait_consumer_end_from_ntb:
sql select * from consumeresult
print ==> rows: $rows
print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
if $rows != 2 then
sleep 1000
goto wait_consumer_end_from_ntb
endi
if $data[0][1] == 0 then
if $data[1][1] != 1 then
return -1
endi
endi
if $data[0][1] == 1 then
if $data[1][1] != 0 then
return -1
endi
endi
# either $data[0][2] $totalMsgOfOneTopic and $data[1][2] == $totalMsgOfNtb
# or $data[0][2] $totalMsgOfNtb and $data[1][2] == $totalMsgOfOneTopic
if $data[0][2] == $totalMsgOfOneTopic then
if $data[1][2] == $totalMsgOfNtb then
goto check_ok_2
endi
elif $data[1][2] == $totalMsgOfOneTopic then
if $data[0][2] == $totalMsgOfNtb then
goto check_ok_2
endi
endi
return -1
check_ok_2:
if $data[0][3] == $totalMsgOfOneTopic then
if $data[1][3] == $totalMsgOfNtb then
goto check_ok_3
endi
elif $data[1][3] == $totalMsgOfOneTopic then
if $data[0][3] == $totalMsgOfNtb then
goto check_ok_3
endi
endi
return -1
check_ok_3:
#------ not need stop consumer, because it exit after pull msg overthan expect msg
#system tsim/tmq/consume.sh -s stop -x SIGINT
system sh/exec.sh -n dnode1 -s stop -x SIGINT

4
tests/script/tsim/tmq/basic3.sim
View File

@ -175,7 +175,7 @@ $totalMsgOfCtb = $rowsPerCtb
$expectmsgcnt = $totalMsgOfCtb
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
print == start consumer to pull msgs from stb
print == start consumer to pull msgs from ctb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
@ -246,7 +246,7 @@ $totalMsgOfNtb = $rowsPerCtb
$expectmsgcnt = $totalMsgOfNtb
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
print == start consumer to pull msgs from stb
print == start consumer to pull msgs from ntb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start

102
tests/script/tsim/tmq/basic3Of2Cons.sim
View File

@ -145,6 +145,12 @@ if $data[1][2] >= $expectmsgcnt then
return -1
endi
$sumOfMsgCnt = $data[0][2] + $data[1][2]
if $sumOfMsgCnt != $expectmsgcnt then
return -1
endi
if $data[0][3] <= 0 then
return -1
endi
@ -158,6 +164,12 @@ endi
if $data[1][3] >= $expectmsgcnt then
return -1
endi
$sumOfMsgRows = $data[0][3] + $data[1][3]
if $sumOfMsgRows != $expectmsgcnt then
return -1
endi
$loop_cnt = $loop_cnt + 1
goto loop_consume_diff_topic_from_stb
loop_consume_diff_topic_from_stb_end:
@ -234,39 +246,32 @@ if $data[0][1] == 1 then
endi
endi
if $data[0][2] != $totalMsgOfCtb then
if $data[1][2] != $totalMsgOfCtb then
return -1
# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0
# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb
if $data[0][2] == $totalMsgOfCtb then
if $data[1][2] == 0 then
goto check_ok_0
endi
if $data[0][2] != 0 then
return -1
endi
endi
if $data[1][2] != $totalMsgOfCtb then
if $data[0][2] != $totalMsgOfCtb then
return -1
endi
if $data[1][2] != 0 then
return -1
elif $data[1][2] == $totalMsgOfCtb then
if $data[0][2] == 0 then
goto check_ok_0
endi
endi
return -1
check_ok_0:
if $data[0][3] != $totalMsgOfCtb then
if $data[1][3] != $totalMsgOfCtb then
return -1
if $data[0][3] == $totalMsgOfCtb then
if $data[1][3] == 0 then
goto check_ok_1
endi
if $data[0][3] != 0 then
return -1
endi
endi
if $data[1][3] != $totalMsgOfCtb then
if $data[0][3] != $totalMsgOfCtb then
return -1
elif $data[1][3] == $totalMsgOfCtb then
if $data[0][3] == 0 then
goto check_ok_1
endi
if $data[1][3] != 0 then
return -1
endi
endi
return -1
check_ok_1:
$loop_cnt = $loop_cnt + 1
goto loop_consume_diff_topic_from_ctb
loop_consume_diff_topic_from_ctb_end:
@ -343,39 +348,32 @@ if $data[1][1] == 0 then
endi
endi
if $data[0][2] != $totalMsgOfNtb then
if $data[1][2] != $totalMsgOfNtb then
return -1
# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0
# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb
if $data[0][2] == $totalMsgOfNtb then
if $data[1][2] == 0 then
goto check_ok_2
endi
if $data[0][2] != 0 then
return -1
endi
endi
if $data[1][2] != $totalMsgOfNtb then
if $data[0][2] != $totalMsgOfNtb then
return -1
endi
if $data[1][2] != 0 then
return -1
elif $data[1][2] == $totalMsgOfNtb then
if $data[0][2] == 0 then
goto check_ok_2
endi
endi
return -1
check_ok_2:
if $data[0][3] != $totalMsgOfNtb then
if $data[1][3] != $totalMsgOfNtb then
return -1
if $data[0][3] == $totalMsgOfNtb then
if $data[1][3] == 0 then
goto check_ok_3
endi
if $data[0][3] != 0 then
return -1
endi
endi
if $data[1][3] != $totalMsgOfNtb then
if $data[0][3] != $totalMsgOfNtb then
return -1
elif $data[1][3] == $totalMsgOfNtb then
if $data[0][3] == 0 then
goto check_ok_3
endi
if $data[1][3] != 0 then
return -1
endi
endi
return -1
check_ok_3:
$loop_cnt = $loop_cnt + 1
goto loop_consume_diff_topic_from_ntb
loop_consume_diff_topic_from_ntb_end:

4
tests/script/tsim/tmq/basic4.sim
View File

@ -130,7 +130,7 @@ $totalMsgOfCtb = $rowsPerCtb * $topicNum
$expectmsgcnt = $totalMsgOfCtb
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
print == start consumer to pull msgs from stb
print == start consumer to pull msgs from ctb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
@ -187,7 +187,7 @@ $totalMsgOfNtb = $rowsPerCtb * $topicNum
$expectmsgcnt = $totalMsgOfNtb
sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
print == start consumer to pull msgs from stb
print == start consumer to pull msgs from ntb
print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start

99
tests/script/tsim/tmq/basic4Of2Cons.sim
View File

@ -114,6 +114,11 @@ if $data[1][2] >= $expectmsgcnt then
return -1
endi
$sumOfConsMsg = $data[0][2] + $data[1][2]
if $sumOfConsMsg != $expectmsgcnt then
return -1
endi
if $data[0][3] <= 0 then
return -1
endi
@ -128,6 +133,11 @@ if $data[1][3] >= $expectmsgcnt then
return -1
endi
$sumOfConsRow = $data[0][3] + $data[1][3]
if $sumOfConsRow != $expectmsgcnt then
return -1
endi
#######################################################################################
# clear consume info and consume result
#run tsim/tmq/clearConsume.sim
@ -189,39 +199,32 @@ if $data[0][1] == 1 then
endi
endi
if $data[0][2] != $totalMsgOfCtb then
if $data[1][2] != $totalMsgOfCtb then
return -1
# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0
# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb
if $data[0][2] == $totalMsgOfCtb then
if $data[1][2] == 0 then
goto check_ok_0
endi
if $data[0][2] != 0 then
return -1
endi
endi
if $data[1][2] != $totalMsgOfCtb then
if $data[0][2] != $totalMsgOfCtb then
return -1
endi
if $data[1][2] != 0 then
return -1
elif $data[0][2] == 0 then
if $data[1][2] == $totalMsgOfCtb then
goto check_ok_0
endi
endi
return -1
check_ok_0:
if $data[0][3] != $totalMsgOfCtb then
if $data[1][3] != $totalMsgOfCtb then
return -1
if $data[0][3] == $totalMsgOfCtb then
if $data[1][3] == 0 then
goto check_ok_1
endi
if $data[0][3] != 0 then
return -1
endi
endi
if $data[1][3] != $totalMsgOfCtb then
if $data[0][3] != $totalMsgOfCtb then
return -1
elif $data[0][3] == 0 then
if $data[1][3] == $totalMsgOfCtb then
goto check_ok_1
endi
if $data[1][3] != 0 then
return -1
endi
endi
return -1
check_ok_1:
#######################################################################################
# clear consume info and consume result
@ -284,39 +287,31 @@ if $data[1][1] == 0 then
endi
endi
if $data[0][2] != $totalMsgOfNtb then
if $data[1][2] != $totalMsgOfNtb then
return -1
# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0
# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb
if $data[0][2] == $totalMsgOfNtb then
if $data[1][2] == 0 then
goto check_ok_2
endi
if $data[0][2] != 0 then
return -1
endi
endi
if $data[1][2] != $totalMsgOfNtb then
if $data[0][2] != $totalMsgOfNtb then
return -1
endi
if $data[1][2] != 0 then
return -1
elif $data[0][2] == 0 then
if $data[1][2] == $totalMsgOfNtb then
goto check_ok_2
endi
endi
return -1
check_ok_2:
if $data[0][3] != $totalMsgOfNtb then
if $data[1][3] != $totalMsgOfNtb then
return -1
if $data[0][3] == $totalMsgOfNtb then
if $data[1][3] == 0 then
goto check_ok_3
endi
if $data[0][3] != 0 then
return -1
endi
endi
if $data[1][3] != $totalMsgOfNtb then
if $data[0][3] != $totalMsgOfNtb then
return -1
elif $data[0][3] == 0 then
if $data[1][3] == $totalMsgOfNtb then
goto check_ok_3
endi
if $data[1][3] != 0 then
return -1
endi
endi
return -1
check_ok_3:
#------ not need stop consumer, because it exit after pull msg overthan expect msg
#system tsim/tmq/consume.sh -s stop -x SIGINT

161
tests/system-test/2-query/cast.py
View File

@ -6,6 +6,7 @@ import inspect
from util.log import *
from util.sql import *
from util.cases import *
from util.dnodes import *
@ -15,56 +16,50 @@ class TDTestCase:
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor())
def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring
tdSql.prepare()
def __cast_to_bigint(self, col_name, tbname):
__sql = f"select cast({col_name} as bigint), {col_name} from {tbname}"
tdSql.query(sql=__sql)
data_tb_col = [result[1] for result in tdSql.queryResult]
for i in range(len(tdSql.queryRows)):
tdSql.checkData( i, 0, None ) if data_tb_col[i] is None else tdSql.checkData( i, 0, int(data_tb_col[i]) )
tdLog.printNoPrefix("==========step1:create table")
tdSql.execute(
'''create table stb1
(ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
tags (t1 int)
'''
)
tdSql.execute(
'''
create table t1
(ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
'''
)
for i in range(4):
tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
def __range_to_bigint(self,cols,tables):
for col in cols:
for table in tables:
self.__cast_to_bigint(col_name=col, tbname=table)
tdLog.printNoPrefix("==========step2:insert data")
for i in range(9):
tdSql.execute(
f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
)
tdSql.execute(
f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
)
tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )")
tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )")
def __cast_to_timestamp(self, col_name, tbname):
__sql = f"select cast({col_name} as timestamp), {col_name} from {tbname}"
tdSql.query(sql=__sql)
data_tb_col = [result[1] for result in tdSql.queryResult]
for i in range(len(tdSql.queryRows)):
if data_tb_col[i] is None:
tdSql.checkData( i, 0 , None )
if (col_name == "c2" or col_name == "double" ) and tbname == "t1" and i == 10:
continue
else:
utc_zone = datetime.timezone.utc
utc_8 = datetime.timezone(datetime.timedelta(hours=8))
date_init_stamp = datetime.datetime.utcfromtimestamp(data_tb_col[i]/1000)
date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f")
tdSql.checkData( i, 0, date_data)
tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
def __range_to_timestamp(self, cols, tables):
for col in cols:
for table in tables:
self.__cast_to_timestamp(col_name=col, tbname=table)
tdSql.execute(
f'''insert into t1 values
( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a )
( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a )
( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a )
( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a )
( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a )
( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a )
( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" )
( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" )
( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" )
( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
'''
)
def __test_bigint(self):
__table_list = ["ct1", "ct4", "t1"]
__col_list = ["c1","c2","c3","c4","c5","c6","c7","c10","c1+c2"]
self.__range_to_bigint(cols=__col_list, tables=__table_list)
def __test_timestamp(self):
__table_list = ["ct1", "ct4", "t1"]
__col_list = ["c1","c2","c3","c4","c5","c6","c7","c1+c2"]
self.__range_to_timestamp(cols=__col_list, tables=__table_list)
def all_test(self):
tdSql.query("select c1 from ct4")
data_ct4_c1 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)]
@ -82,9 +77,9 @@ class TDTestCase:
tdLog.printNoPrefix("==========step5: cast int to binary, expect changes to str(int) ")
tdSql.query("select cast(c1 as binary(32)) as b from ct4")
for i in range(len(data_ct4_c1)):
tdSql.checkData( i, 0, str(data_ct4_c1[i]) )
#tdSql.query("select cast(c1 as binary(32)) as b from ct4")
#for i in range(len(data_ct4_c1)):
# tdSql.checkData( i, 0, str(data_ct4_c1[i]) )
tdSql.query("select cast(c1 as binary(32)) as b from t1")
for i in range(len(data_t1_c1)):
tdSql.checkData( i, 0, str(data_t1_c1[i]) )
@ -240,7 +235,7 @@ class TDTestCase:
tdSql.checkData( i, 0, date_data)
tdLog.printNoPrefix("==========step16: cast smallint to bigint, expect no changes")
tdLog.printNoPrefix("==========step16: cast tinyint to bigint, expect no changes")
tdSql.query("select c4 from ct4")
data_ct4_c4 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)]
tdSql.query("select c4 from t1")
@ -254,7 +249,7 @@ class TDTestCase:
tdSql.checkData( i, 0, data_t1_c4[i])
tdLog.printNoPrefix("==========step17: cast smallint to binary, expect changes to str(int) ")
tdLog.printNoPrefix("==========step17: cast tinyint to binary, expect changes to str(int) ")
tdSql.query("select cast(c4 as binary(32)) as b from ct4")
for i in range(len(data_ct4_c4)):
@ -263,7 +258,7 @@ class TDTestCase:
for i in range(len(data_t1_c4)):
tdSql.checkData( i, 0, str(data_t1_c4[i]) )
tdLog.printNoPrefix("==========step18: cast smallint to nchar, expect changes to str(int) ")
tdLog.printNoPrefix("==========step18: cast tinyint to nchar, expect changes to str(int) ")
tdSql.query("select cast(c4 as nchar(32)) as b from ct4")
for i in range(len(data_ct4_c4)):
@ -272,7 +267,7 @@ class TDTestCase:
for i in range(len(data_t1_c4)):
tdSql.checkData( i, 0, str(data_t1_c4[i]) )
tdLog.printNoPrefix("==========step19: cast smallint to timestamp, expect changes to timestamp ")
tdLog.printNoPrefix("==========step19: cast tinyint to timestamp, expect changes to timestamp ")
tdSql.query("select cast(c4 as timestamp) as b from ct4")
for i in range(len(data_ct4_c4)):
@ -624,7 +619,67 @@ class TDTestCase:
tdSql.error("select cast(c8 as timestamp ) as b from ct4")
tdSql.error("select cast(c9 as timestamp ) as b from ct4")
tdSql.error("select cast(c9 as binary(64) ) as b from ct4")
pass
def run(self):
tdSql.prepare()
tdLog.printNoPrefix("==========step1:create table")
tdSql.execute(
'''create table stb1
(ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
tags (t1 int)
'''
)
tdSql.execute(
'''
create table t1
(ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
'''
)
for i in range(4):
tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
tdLog.printNoPrefix("==========step2:insert data")
for i in range(9):
tdSql.execute(
f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
)
tdSql.execute(
f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
)
tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )")
tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )")
tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute(
f'''insert into t1 values
( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a )
( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a )
( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a )
( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a )
( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a )
( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a )
( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" )
( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" )
( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" )
( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
'''
)
self.all_test()
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.execute("use db")
self.all_test()
def stop(self):
tdSql.close()

232
tests/system-test/2-query/sum.py Normal file
View File

@ -0,0 +1,232 @@
from util.log import *
from util.sql import *
from util.cases import *
from util.dnodes import *
INT_COL = "c1"
BINT_COL = "c2"
SINT_COL = "c3"
TINT_COL = "c4"
FLOAT_COL = "c5"
DOUBLE_COL = "c6"
BOOL_COL = "c7"
BINARY_COL = "c8"
NCHAR_COL = "c9"
TS_COL = "c10"
NUM_COL = [INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ]
UN_NUM_COL = [BOOL_COL, BINARY_COL, NCHAR_COL, ]
TS_TYPE_COL = [TS_COL]
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor())
def __sum_condition(self):
sum_condition = []
for num_col in NUM_COL:
sum_condition.extend(
(
num_col,
f"ceil( {num_col} )",
)
)
sum_condition.extend( f"{num_col} + {num_col_2}" for num_col_2 in NUM_COL )
sum_condition.extend( f"{num_col} + {un_num_col} " for un_num_col in UN_NUM_COL )
sum_condition.append(1)
return sum_condition
def __where_condition(self, col):
return f" where abs( {col} ) < 1000000 "
def __group_condition(self, col, having = ""):
return f" group by {col} having {having}" if having else f" group by {col} "
def __sum_current_check(self, tbname):
sum_condition = self.__sum_condition()
for condition in sum_condition:
where_condition = self.__where_condition(condition)
group_condition = self.__group_condition(condition, having=f"{condition} is not null " )
tdSql.query(f"select {condition} from {tbname} {where_condition} ")
datas = [tdSql.getData(i,0) for i in range(tdSql.queryRows)]
sum_data = sum(filter(None, datas))
tdSql.query(f"select sum( {condition} ) from {tbname} {where_condition} ")
tdSql.checkData(0, 0, sum_data)
tdSql.query(f"select {condition} from {tbname} {where_condition} {group_condition} ")
def __sum_err_check(self,tbanme):
sqls = []
for un_num_col in UN_NUM_COL:
sqls.extend(
(
f"select sum( {un_num_col} ) from {tbanme} ",
f"select sum(ceil( {un_num_col} )) from {tbanme} ",
)
)
sqls.extend( f"select sum( {un_num_col} + {un_num_col_2} ) from {tbanme} " for un_num_col_2 in UN_NUM_COL )
sqls.extend( f"select sum( {num_col} + {ts_col} ) from {tbanme} " for num_col in NUM_COL for ts_col in TS_TYPE_COL)
sqls.extend(
(
f"select sum() from {tbanme} ",
f"select sum(*) from {tbanme} ",
f"select sum(ccccccc) from {tbanme} ",
f"select sum('test') from {tbanme} ",
)
)
return sqls
def __test_current(self):
tdLog.printNoPrefix("==========current sql condition check , must return query ok==========")
tbname = ["ct1", "ct2", "ct4", "t1"]
for tb in tbname:
self.__sum_current_check(tb)
tdLog.printNoPrefix(f"==========current sql condition check in {tb} over==========")
def __test_error(self):
tdLog.printNoPrefix("==========err sql condition check , must return error==========")
tbname = ["ct1", "ct2", "ct4", "t1"]
for tb in tbname:
for errsql in self.__sum_err_check(tb):
tdSql.error(sql=errsql)
tdLog.printNoPrefix(f"==========err sql condition check in {tb} over==========")
def all_test(self):
self.__test_current()
self.__test_error()
def __create_tb(self):
tdSql.prepare()
tdLog.printNoPrefix("==========step1:create table")
create_stb_sql = f'''create table stb1(
ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint,
{FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool,
{BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp
) tags (t1 int)
'''
create_ntb_sql = f'''create table t1(
ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint,
{FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool,
{BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp
)
'''
tdSql.execute(create_stb_sql)
tdSql.execute(create_ntb_sql)
for i in range(4):
tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
def __insert_data(self, rows):
for i in range(9):
tdSql.execute(
f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
)
tdSql.execute(
f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
)
tdSql.execute(
f"insert into ct2 values ( now()-{i*90}d, {-1*i}, {-11111*i}, {-111*i}, {-11*i}, {-1.11*i}, {-11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
)
tdSql.execute(
'''insert into ct1 values
( now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )
( now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )
'''
)
tdSql.execute(
f'''insert into ct4 values
( now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
( now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
( now()+{rows * 9}d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
(
now()+{rows * 9-10}d, {pow(2,31)-pow(2,15)}, {pow(2,63)-pow(2,30)}, 32767, 127,
{ 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nachar_limit-1", now()-1d
)
(
now()+{rows * 9-20}d, {pow(2,31)-pow(2,16)}, {pow(2,63)-pow(2,31)}, 32766, 126,
{ 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nachar_limit-2", now()-2d
)
'''
)
tdSql.execute(
f'''insert into ct2 values
( now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
( now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
( now()+{rows * 9}d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
(
now()+{rows * 9-10}d, { -1 * pow(2,31) + pow(2,15) }, { -1 * pow(2,63) + pow(2,30) }, -32766, -126,
{ -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nachar_limit-1", now()-1d
)
(
now()+{rows * 9-20}d, { -1 * pow(2,31) + pow(2,16) }, { -1 * pow(2,63) + pow(2,31) }, -32767, -127,
{ - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nachar_limit-2", now()-2d
)
'''
)
for i in range(rows):
insert_data = f'''insert into t1 values
( now()-{i}h, {i}, {i}, { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2},
"binary_{i}", "nchar_{i}", now()-{i}s )
'''
tdSql.execute(insert_data)
tdSql.execute(
f'''insert into t1 values
( now() + 3h, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
( now()-{ ( rows // 2 ) * 60 + 30 }m, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
( now()-{rows}h, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
( now() + 2h, { pow(2,31) - pow(2,15) }, { pow(2,63) - pow(2,30) }, 32767, 127,
{ 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 },
"binary_limit-1", "nachar_limit-1", now()-1d
)
(
now() + 1h , { pow(2,31) - pow(2,16) }, { pow(2,63) - pow(2,31) }, 32766, 126,
{ 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 },
"binary_limit-2", "nachar_limit-2", now()-2d
)
'''
)
def run(self):
tdSql.prepare()
tdLog.printNoPrefix("==========step1:create table")
self.__create_tb()
tdLog.printNoPrefix("==========step2:insert data")
self.__insert_data(100)
tdLog.printNoPrefix("==========step3:all check")
self.all_test()
# tdDnodes.stop(1)
# tdDnodes.start(1)
# tdSql.execute("use db")
# tdLog.printNoPrefix("==========step4:after wal, all check again ")
# self.all_test()
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

7
tests/system-test/fulltest.sh
View File

@ -2,9 +2,12 @@
set -e
#python3 ./test.py -f 2-query/between.py
python3 ./test.py -f 2-query/distinct.py
#python3 ./test.py -f 2-query/distinct.py
python3 ./test.py -f 2-query/varchar.py
python3 ./test.py -f 2-query/cast.py
python3 ./test.py -f 2-query/timezone.py
python3 ./test.py -f 2-query/Now.py
python3 ./test.py -f 2-query/Today.py
#python3 ./test.py -f 2-query/cast.py

8
tests/test/c/tmqSim.c
View File

@ -125,13 +125,13 @@ void saveConfigToLogFile() {
for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) {
taosFprintfFile(g_fp, "# consumer %d info:\n", g_stConfInfo.stThreads[i].consumerId);
taosFprintfFile(g_fp, " Topics: ");
for (int i = 0 ; i < g_stConfInfo.stThreads[i].numOfTopic; i++) {
taosFprintfFile(g_fp, "%s, ", g_stConfInfo.stThreads[i].topics[i]);
for (int j = 0 ; j < g_stConfInfo.stThreads[i].numOfTopic; j++) {
taosFprintfFile(g_fp, "%s, ", g_stConfInfo.stThreads[i].topics[j]);
}
taosFprintfFile(g_fp, "\n");
taosFprintfFile(g_fp, " Key: ");
for (int i = 0 ; i < g_stConfInfo.stThreads[i].numOfKey; i++) {
taosFprintfFile(g_fp, "%s:%s, ", g_stConfInfo.stThreads[i].key[i], g_stConfInfo.stThreads[i].value[i]);
for (int k = 0 ; k < g_stConfInfo.stThreads[i].numOfKey; k++) {
taosFprintfFile(g_fp, "%s:%s, ", g_stConfInfo.stThreads[i].key[k], g_stConfInfo.stThreads[i].value[k]);
}
taosFprintfFile(g_fp, "\n");
}

93
tools/shell/inc/shell.h
View File

@ -1,93 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_SHELL_H_
#define _TD_SHELL_H_
#include "os.h"
#include "taos.h"
#include "taosdef.h"
#define MAX_USERNAME_SIZE 64
#define MAX_DBNAME_SIZE 64
#define MAX_IP_SIZE 20
#define MAX_HISTORY_SIZE 1000
#define MAX_COMMAND_SIZE 1048586
#define HISTORY_FILE ".taos_history"
#define DEFAULT_RES_SHOW_NUM 100
typedef struct SShellHistory {
char* hist[MAX_HISTORY_SIZE];
int hstart;
int hend;
} SShellHistory;
typedef struct SShellArguments {
char* host;
char* password;
char* user;
char* auth;
char* database;
char* timezone;
bool is_raw_time;
bool is_use_passwd;
bool dump_config;
char file[TSDB_FILENAME_LEN];
char dir[TSDB_FILENAME_LEN];
int threadNum;
int check;
bool status;
bool verbose;
char* commands;
int abort;
int port;
int pktLen;
int pktNum;
char* pktType;
char* netTestRole;
} SShellArguments;
/**************** Function declarations ****************/
extern void shellParseArgument(int argc, char* argv[], SShellArguments* arguments);
extern TAOS* shellInit(SShellArguments* args);
extern void* shellLoopQuery(void* arg);
extern void taos_error(TAOS_RES* tres, int64_t st);
extern int regex_match(const char* s, const char* reg, int cflags);
int32_t shellReadCommand(TAOS* con, char command[]);
int32_t shellRunCommand(TAOS* con, char* command);
void shellRunCommandOnServer(TAOS* con, char command[]);
void read_history();
void write_history();
void source_file(TAOS* con, char* fptr);
void source_dir(TAOS* con, SShellArguments* args);
void get_history_path(char* history);
void shellCheck(TAOS* con, SShellArguments* args);
void cleanup_handler(void* arg);
void exitShell();
int shellDumpResult(TAOS_RES* con, char* fname, int* error_no, bool printMode);
void shellGetGrantInfo(void *con);
int isCommentLine(char *line);
/**************** Global variable declarations ****************/
extern char PROMPT_HEADER[];
extern char CONTINUE_PROMPT[];
extern int prompt_size;
extern SShellHistory history;
extern SShellArguments args;
extern int64_t result;
#endif

55
tools/shell/inc/shellCommand.h
View File

@ -1,55 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_SHELL_COMMAND_H_
#define _TD_SHELL_COMMAND_H_
#include "shell.h"
#define LEFT 1
#define RIGHT 2
#define UP 3
#define DOWN 4
typedef struct Command Command;
struct Command {
char * buffer;
char * command;
unsigned commandSize;
unsigned bufferSize;
unsigned cursorOffset;
unsigned screenOffset;
unsigned endOffset;
};
extern void backspaceChar(Command *cmd);
extern void clearLineBefore(Command *cmd);
extern void clearLineAfter(Command *cmd);
extern void deleteChar(Command *cmd);
extern void moveCursorLeft(Command *cmd);
extern void moveCursorRight(Command *cmd);
extern void positionCursorHome(Command *cmd);
extern void positionCursorEnd(Command *cmd);
extern void showOnScreen(Command *cmd);
extern void updateBuffer(Command *cmd);
extern int isReadyGo(Command *cmd);
extern void resetCommand(Command *cmd, const char s[]);
int countPrefixOnes(unsigned char c);
void clearScreen(int ecmd_pos, int cursor_pos);
void printChar(char c, int times);
void positionCursor(int step, int direction);
#endif

117
tools/shell/inc/shellInt.h Normal file
View File

@ -0,0 +1,117 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_SHELL_INT_H_
#define _TD_SHELL_INT_H_
#include "os.h"
#include "taos.h"
#include "taosdef.h"
#include "taoserror.h"
#include "tconfig.h"
#include "tglobal.h"
#include "trpc.h"
#include "ttypes.h"
#include "tutil.h"
#define SHELL_MAX_HISTORY_SIZE 1000
#define SHELL_MAX_COMMAND_SIZE 1048586
#define SHELL_HISTORY_FILE ".taos_history"
#define SHELL_DEFAULT_RES_SHOW_NUM 100
#define SHELL_DEFAULT_MAX_BINARY_DISPLAY_WIDTH 30
#define SHELL_MAX_PKG_LEN 2 * 1024 * 1024
#define SHELL_MIN_PKG_LEN 1
#define SHELL_DEF_PKG_LEN 1024
#define SHELL_MAX_PKG_NUM 1 * 1024 * 1024
#define SHELL_MIN_PKG_NUM 1
#define SHELL_DEF_PKG_NUM 100
typedef struct {
char* hist[SHELL_MAX_HISTORY_SIZE];
char file[TSDB_FILENAME_LEN];
int32_t hstart;
int32_t hend;
} SShellHistory;
typedef struct {
const char* host;
const char* user;
const char* auth;
const char* database;
const char* cfgdir;
const char* commands;
const char* netrole;
char file[PATH_MAX];
char password[TSDB_USET_PASSWORD_LEN];
bool is_gen_auth;
bool is_raw_time;
bool is_version;
bool is_dump_config;
bool is_check;
bool is_startup;
bool is_help;
uint16_t port;
int32_t pktLen;
int32_t pktNum;
int32_t displayWidth;
int32_t abort;
} SShellArgs;
typedef struct {
const char* clientVersion;
const char* promptHeader;
const char* promptContinue;
const char* osname;
int32_t promptSize;
char programVersion[32];
} SShellOsDetails;
typedef struct {
SShellArgs args;
SShellHistory history;
SShellOsDetails info;
TAOS* conn;
TdThread pid;
tsem_t cancelSem;
int64_t result;
} SShellObj;
// shellArguments.c
int32_t shellParseArgs(int32_t argc, char* argv[]);
// shellCommand.c
int32_t shellReadCommand(char* command);
// shellEngine.c
int32_t shellExecute();
// shellUtil.c
int32_t shellCheckIntSize();
void shellPrintVersion();
void shellPrintHelp();
void shellGenerateAuth();
void shellDumpConfig();
void shellCheckServerStatus();
bool shellRegexMatch(const char* s, const char* reg, int32_t cflags);
void shellExit();
// shellNettest.c
void shellTestNetWork();
// shellMain.c
extern SShellObj shell;
extern void taos_init();
#endif /*_TD_SHELL_INT_H_*/

141
tools/shell/inc/syncMsg.h
View File

@ -1,141 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_SYNC_MSG_H
#define TDENGINE_SYNC_MSG_H
#ifdef __cplusplus
extern "C" {
#endif
#include "tsync.h"
typedef enum {
TAOS_SMSG_START = 0,
TAOS_SMSG_SYNC_DATA = 1,
TAOS_SMSG_SYNC_DATA_RSP = 2,
TAOS_SMSG_SYNC_FWD = 3,
TAOS_SMSG_SYNC_FWD_RSP = 4,
TAOS_SMSG_SYNC_REQ = 5,
TAOS_SMSG_SYNC_REQ_RSP = 6,
TAOS_SMSG_SYNC_MUST = 7,
TAOS_SMSG_SYNC_MUST_RSP = 8,
TAOS_SMSG_STATUS = 9,
TAOS_SMSG_STATUS_RSP = 10,
TAOS_SMSG_SETUP = 11,
TAOS_SMSG_SETUP_RSP = 12,
TAOS_SMSG_SYNC_FILE = 13,
TAOS_SMSG_SYNC_FILE_RSP = 14,
TAOS_SMSG_TEST = 15,
TAOS_SMSG_END = 16
} ESyncMsgType;
typedef enum {
SYNC_STATUS_BROADCAST,
SYNC_STATUS_BROADCAST_RSP,
SYNC_STATUS_SETUP_CONN,
SYNC_STATUS_SETUP_CONN_RSP,
SYNC_STATUS_EXCHANGE_DATA,
SYNC_STATUS_EXCHANGE_DATA_RSP,
SYNC_STATUS_CHECK_ROLE,
SYNC_STATUS_CHECK_ROLE_RSP
} ESyncStatusType;
#pragma pack(push, 1)
typedef struct {
int8_t type; // msg type
int8_t protocol; // protocol version
uint16_t signature; // fixed value
int32_t code; //
int32_t cId; // cluster Id
int32_t vgId; // vg ID
int32_t len; // content length, does not include head
uint32_t cksum;
} SSyncHead;
typedef struct {
SSyncHead head;
uint16_t port;
uint16_t tranId;
int32_t sourceId; // only for arbitrator
char fqdn[TSDB_FQDN_LEN];
} SSyncMsg;
typedef struct {
SSyncHead head;
int8_t sync;
int8_t reserved;
uint16_t tranId;
int8_t reserverd[4];
} SSyncRsp;
typedef struct {
int8_t role;
uint64_t version;
} SPeerStatus;
typedef struct {
SSyncHead head;
int8_t role;
int8_t ack;
int8_t type;
int8_t reserved[3];
uint16_t tranId;
uint64_t version;
SPeerStatus peersStatus[TAOS_SYNC_MAX_REPLICA];
} SPeersStatus;
typedef struct {
SSyncHead head;
uint64_t fversion;
} SFileVersion;
typedef struct {
SSyncHead head;
int8_t ack;
} SFileAck;
typedef struct {
SSyncHead head;
uint64_t version;
int32_t code;
} SFwdRsp;
#pragma pack(pop)
#define SYNC_PROTOCOL_VERSION 1
#define SYNC_SIGNATURE ((uint16_t)(0xCDEF))
extern char *statusType[];
uint16_t syncGenTranId();
int32_t syncCheckHead(SSyncHead *pHead);
void syncBuildSyncFwdMsg(SSyncHead *pHead, int32_t vgId, int32_t len);
void syncBuildSyncFwdRsp(SFwdRsp *pMsg, int32_t vgId, uint64_t version, int32_t code);
void syncBuildSyncReqMsg(SSyncMsg *pMsg, int32_t vgId);
void syncBuildSyncDataMsg(SSyncMsg *pMsg, int32_t vgId);
void syncBuildSyncSetupMsg(SSyncMsg *pMsg, int32_t vgId);
void syncBuildPeersStatus(SPeersStatus *pMsg, int32_t vgId);
void syncBuildSyncTestMsg(SSyncMsg *pMsg, int32_t vgId);
void syncBuildFileAck(SFileAck *pMsg, int32_t vgId);
void syncBuildFileVersion(SFileVersion *pMsg, int32_t vgId);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_VNODEPEER_H

127
tools/shell/inc/tsync.h
View File

@ -1,127 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_SYNC_H
#define TDENGINE_SYNC_H
#ifdef __cplusplus
extern "C" {
#endif
#define TAOS_SYNC_MAX_REPLICA 5
#define TAOS_SYNC_MAX_INDEX 0x7FFFFFFF
typedef enum {
TAOS_SYNC_ROLE_OFFLINE = 0,
TAOS_SYNC_ROLE_UNSYNCED = 1,
TAOS_SYNC_ROLE_SYNCING = 2,
TAOS_SYNC_ROLE_SLAVE = 3,
TAOS_SYNC_ROLE_MASTER = 4
} ESyncRole;
typedef enum {
TAOS_SYNC_STATUS_INIT = 0,
TAOS_SYNC_STATUS_START = 1,
TAOS_SYNC_STATUS_FILE = 2,
TAOS_SYNC_STATUS_CACHE = 3
} ESyncStatus;
typedef struct {
uint32_t nodeId; // node ID assigned by TDengine
uint16_t nodePort; // node sync Port
char nodeFqdn[TSDB_FQDN_LEN]; // node FQDN
} SNodeInfo;
typedef struct {
int8_t quorum; // number of confirms required, >=1
int8_t replica; // number of replications, >=1
SNodeInfo nodeInfo[TAOS_SYNC_MAX_REPLICA];
} SSyncCfg;
typedef struct {
int32_t selfIndex;
uint32_t nodeId[TAOS_SYNC_MAX_REPLICA];
int32_t role[TAOS_SYNC_MAX_REPLICA];
} SNodesRole;
// get the wal file from index or after
// return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file
typedef int32_t (*FGetWalInfo)(int32_t vgId, char *fileName, int64_t *fileId);
// when a forward pkt is received, call this to handle data
typedef int32_t (*FWriteToCache)(int32_t vgId, void *pHead, int32_t qtype, void *pMsg);
// when forward is confirmed by peer, master call this API to notify app
typedef void (*FConfirmForward)(int32_t vgId, void *mhandle, int32_t code);
// when role is changed, call this to notify app
typedef void (*FNotifyRole)(int32_t vgId, int8_t role);
// if a number of retrieving data failed, call this to start flow control
typedef void (*FNotifyFlowCtrl)(int32_t vgId, int32_t level);
// when data file is synced successfully, notity app
typedef void (*FStartSyncFile)(int32_t vgId);
typedef void (*FStopSyncFile)(int32_t vgId, uint64_t fversion);
// get file version
typedef int32_t (*FGetVersion)(int32_t vgId, uint64_t *fver, uint64_t *vver);
typedef int32_t (*FSendFile)(void *tsdb, SOCKET socketFd);
typedef int32_t (*FRecvFile)(void *tsdb, SOCKET socketFd);
typedef struct {
int32_t vgId; // vgroup ID
uint64_t version; // initial version
SSyncCfg syncCfg; // configuration from mgmt
char path[TSDB_FILENAME_LEN]; // path to the file
void * pTsdb;
FGetWalInfo getWalInfoFp;
FWriteToCache writeToCacheFp;
FConfirmForward confirmForward;
FNotifyRole notifyRoleFp;
FNotifyFlowCtrl notifyFlowCtrlFp;
FStartSyncFile startSyncFileFp;
FStopSyncFile stopSyncFileFp;
FGetVersion getVersionFp;
FSendFile sendFileFp;
FRecvFile recvFileFp;
} SSyncInfo;
typedef void *tsync_h;
int32_t syncInit();
void syncCleanUp();
int64_t syncStart(const SSyncInfo *);
void syncStop(int64_t rid);
int32_t syncReconfig(int64_t rid, const SSyncCfg *);
int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int32_t qtype, bool force);
void syncConfirmForward(int64_t rid, uint64_t version, int32_t code, bool force);
void syncRecover(int64_t rid); // recover from other nodes:
int32_t syncGetNodesRole(int64_t rid, SNodesRole *);
extern char *syncRole[];
//global configurable parameters
extern int32_t sDebugFlag;
extern char tsArbitrator[];
extern uint16_t tsSyncPort;
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_SYNC_H

202
tools/shell/src/backup/shellCheck.c
View File

@ -1,202 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _GNU_SOURCE
#define _XOPEN_SOURCE
#define _DEFAULT_SOURCE
#include "os.h"
#include "shell.h"
#include "shellCommand.h"
#include "tglobal.h"
#include "tutil.h"
#define SHELL_SQL_LEN 1024
static int32_t tbNum = 0;
static int32_t tbMallocNum = 0;
static char ** tbNames = NULL;
static int32_t checkedNum = 0;
static int32_t errorNum = 0;
typedef struct {
TdThread threadID;
int threadIndex;
int totalThreads;
void * taos;
char * db;
} ShellThreadObj;
static int32_t shellUseDb(TAOS *con, char *db) {
if (db == NULL) {
fprintf(stdout, "no dbname input\n");
return -1;
}
char sql[SHELL_SQL_LEN] = {0};
snprintf(sql, SHELL_SQL_LEN, "use %s", db);
TAOS_RES *pSql = taos_query(con, sql);
int32_t code = taos_errno(pSql);
if (code != 0) {
fprintf(stdout, "failed to execute sql:%s since %s", sql, taos_errstr(pSql));
}
taos_free_result(pSql);
return code;
}
static int32_t shellShowTables(TAOS *con, char *db) {
char sql[SHELL_SQL_LEN] = {0};
snprintf(sql, SHELL_SQL_LEN, "show %s.tables", db);
TAOS_RES *pSql = taos_query(con, sql);
int32_t code = taos_errno(pSql);
if (code != 0) {
fprintf(stdout, "failed to execute sql:%s since %s\n", sql, taos_errstr(pSql));
} else {
TAOS_ROW row;
while ((row = taos_fetch_row(pSql))) {
int32_t tbIndex = tbNum++;
if (tbMallocNum < tbNum) {
tbMallocNum = (tbMallocNum * 2 + 1);
char** tbNames1 = taosMemoryRealloc(tbNames, tbMallocNum * sizeof(char *));
if (tbNames1 == NULL) {
fprintf(stdout, "failed to malloc tablenames, num:%d\n", tbMallocNum);
code = TSDB_CODE_TSC_OUT_OF_MEMORY;
break;
}
tbNames = tbNames1;
}
tbNames[tbIndex] = taosMemoryMalloc(TSDB_TABLE_NAME_LEN);
strncpy(tbNames[tbIndex], (const char *)row[0], TSDB_TABLE_NAME_LEN);
if (tbIndex % 100000 == 0 && tbIndex != 0) {
fprintf(stdout, "%d tablenames fetched\n", tbIndex);
}
}
}
taos_free_result(pSql);
fprintf(stdout, "total %d tablenames fetched, over\n", tbNum);
return code;
}
static void shellFreeTbnames() {
for (int32_t i = 0; i < tbNum; ++i) {
taosMemoryFree(tbNames[i]);
}
taosMemoryFree(tbNames);
}
static void *shellCheckThreadFp(void *arg) {
ShellThreadObj *pThread = (ShellThreadObj *)arg;
setThreadName("shellCheckThrd");
int32_t interval = tbNum / pThread->totalThreads + 1;
int32_t start = pThread->threadIndex * interval;
int32_t end = (pThread->threadIndex + 1) * interval;
if (end > tbNum) end = tbNum + 1;
char file[32] = {0};
snprintf(file, 32, "tb%d.txt", pThread->threadIndex);
TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (!fp) {
fprintf(stdout, "failed to open %s, reason:%s", file, strerror(errno));
return NULL;
}
char sql[SHELL_SQL_LEN];
for (int32_t t = start; t < end; ++t) {
char *tbname = tbNames[t];
if (tbname == NULL) break;
snprintf(sql, SHELL_SQL_LEN, "select * from %s limit 1", tbname);
TAOS_RES *pSql = taos_query(pThread->taos, sql);
int32_t code = taos_errno(pSql);
if (code != 0) {
int32_t len = snprintf(sql, SHELL_SQL_LEN, "drop table %s.%s;\n", pThread->db, tbname);
taosWriteFile(pFile, sql, len);
atomic_add_fetch_32(&errorNum, 1);
}
int32_t cnum = atomic_add_fetch_32(&checkedNum, 1);
if (cnum % 5000 == 0 && cnum != 0) {
fprintf(stdout, "%d tables checked\n", cnum);
}
taos_free_result(pSql);
}
taosFsync(pFile);
taosCloseFile(&pFile);
return NULL;
}
static void shellRunCheckThreads(TAOS *con, SShellArguments *_args) {
TdThreadAttr thattr;
ShellThreadObj *threadObj = (ShellThreadObj *)taosMemoryCalloc(_args->threadNum, sizeof(ShellThreadObj));
for (int t = 0; t < _args->threadNum; ++t) {
ShellThreadObj *pThread = threadObj + t;
pThread->threadIndex = t;
pThread->totalThreads = _args->threadNum;
pThread->taos = con;
pThread->db = _args->database;
taosThreadAttrInit(&thattr);
taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&(pThread->threadID), &thattr, shellCheckThreadFp, (void *)pThread) != 0) {
fprintf(stderr, "ERROR: thread:%d failed to start\n", pThread->threadIndex);
exit(0);
}
}
for (int t = 0; t < _args->threadNum; ++t) {
taosThreadJoin(threadObj[t].threadID, NULL);
}
for (int t = 0; t < _args->threadNum; ++t) {
taos_close(threadObj[t].taos);
}
taosMemoryFree(threadObj);
}
void shellCheck(TAOS *con, SShellArguments *_args) {
int64_t start = taosGetTimestampMs();
if (shellUseDb(con, _args->database) != 0) {
shellFreeTbnames();
return;
}
if (shellShowTables(con, _args->database) != 0) {
shellFreeTbnames();
return;
}
fprintf(stdout, "total %d tables will be checked by %d threads\n", tbNum, _args->threadNum);
shellRunCheckThreads(con, _args);
int64_t end = taosGetTimestampMs();
fprintf(stdout, "total %d tables checked, failed:%d, time spent %.2f seconds\n", checkedNum, errorNum,
(end - start) / 1000.0);
}

498
tools/shell/src/backup/shellDarwin.c
View File

@ -1,498 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define __USE_XOPEN
#include "os.h"
#include "shell.h"
#include "shellCommand.h"
#include "tbase64.h"
#include "tscLog.h"
#define OPT_ABORT 1 /* <20>Cabort */
int indicator = 1;
struct termios oldtio;
void insertChar(Command *cmd, char *c, int size);
void printHelp() {
char indent[10] = " ";
printf("taos shell is used to test the TDengine database\n");
printf("%s%s\n", indent, "-h");
printf("%s%s%s\n", indent, indent, "TDengine server IP address to connect. The default host is localhost.");
printf("%s%s\n", indent, "-p");
printf("%s%s%s\n", indent, indent, "The password to use when connecting to the server.");
printf("%s%s\n", indent, "-P");
printf("%s%s%s\n", indent, indent, "The TCP/IP port number to use for the connection");
printf("%s%s\n", indent, "-u");
printf("%s%s%s\n", indent, indent, "The user name to use when connecting to the server.");
printf("%s%s\n", indent, "-c");
printf("%s%s%s\n", indent, indent, "Configuration directory.");
printf("%s%s\n", indent, "-s");
printf("%s%s%s\n", indent, indent, "Commands to run without enter the shell.");
printf("%s%s\n", indent, "-r");
printf("%s%s%s\n", indent, indent, "Output time as unsigned long..");
printf("%s%s\n", indent, "-f");
printf("%s%s%s\n", indent, indent, "Script to run without enter the shell.");
printf("%s%s\n", indent, "-d");
printf("%s%s%s\n", indent, indent, "Database to use when connecting to the server.");
printf("%s%s\n", indent, "-t");
printf("%s%s%s\n", indent, indent, "Time zone of the shell, default is local.");
printf("%s%s\n", indent, "-D");
printf("%s%s%s\n", indent, indent, "Use multi-thread to import all SQL files in the directory separately.");
printf("%s%s\n", indent, "-T");
printf("%s%s%s\n", indent, indent, "Number of threads when using multi-thread to import data.");
exit(EXIT_SUCCESS);
}
char DARWINCLIENT_VERSION[] = "Welcome to the TDengine shell from %s, Client Version:%s\n"
"Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n";
char g_password[SHELL_MAX_PASSWORD_LEN];
void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) {
wordexp_t full_path;
for (int i = 1; i < argc; i++) {
// for host
if (strcmp(argv[i], "-h") == 0) {
if (i < argc - 1) {
arguments->host = argv[++i];
} else {
fprintf(stderr, "option -h requires an argument\n");
exit(EXIT_FAILURE);
}
}
// for password
else if ((strncmp(argv[i], "-p", 2) == 0)
|| (strncmp(argv[i], "--password", 10) == 0)) {
strcpy(tsOsName, "Darwin");
printf(DARWINCLIENT_VERSION, tsOsName, taos_get_client_info());
if ((strlen(argv[i]) == 2)
|| (strncmp(argv[i], "--password", 10) == 0)) {
printf("Enter password: ");
taosSetConsoleEcho(false);
if (scanf("%s", g_password) > 1) {
fprintf(stderr, "password read error\n");
}
taosSetConsoleEcho(true);
getchar();
} else {
tstrncpy(g_password, (char *)(argv[i] + 2), SHELL_MAX_PASSWORD_LEN);
}
arguments->password = g_password;
arguments->is_use_passwd = true;
strcpy(argv[i], "");
argc -= 1;
}
// for management port
else if (strcmp(argv[i], "-P") == 0) {
if (i < argc - 1) {
arguments->port = atoi(argv[++i]);
} else {
fprintf(stderr, "option -P requires an argument\n");
exit(EXIT_FAILURE);
}
}
// for user
else if (strcmp(argv[i], "-u") == 0) {
if (i < argc - 1) {
arguments->user = argv[++i];
} else {
fprintf(stderr, "option -u requires an argument\n");
exit(EXIT_FAILURE);
}
} else if (strcmp(argv[i], "-c") == 0) {
if (i < argc - 1) {
if (strlen(argv[++i]) >= TSDB_FILENAME_LEN) {
fprintf(stderr, "config file path: %s overflow max len %d\n", argv[i], TSDB_FILENAME_LEN - 1);
exit(EXIT_FAILURE);
}
strcpy(configDir, argv[i]);
} else {
fprintf(stderr, "Option -c requires an argument\n");
exit(EXIT_FAILURE);
}
} else if (strcmp(argv[i], "-s") == 0) {
if (i < argc - 1) {
arguments->commands = argv[++i];
} else {
fprintf(stderr, "option -s requires an argument\n");
exit(EXIT_FAILURE);
}
} else if (strcmp(argv[i], "-r") == 0) {
arguments->is_raw_time = true;
}
// For temperory batch commands to run TODO
else if (strcmp(argv[i], "-f") == 0) {
if (i < argc - 1) {
strcpy(arguments->file, argv[++i]);
} else {
fprintf(stderr, "option -f requires an argument\n");
exit(EXIT_FAILURE);
}
}
// for default database
else if (strcmp(argv[i], "-d") == 0) {
if (i < argc - 1) {
arguments->database = argv[++i];
} else {
fprintf(stderr, "option -d requires an argument\n");
exit(EXIT_FAILURE);
}
}
// For time zone
else if (strcmp(argv[i], "-t") == 0) {
if (i < argc - 1) {
arguments->timezone = argv[++i];
} else {
fprintf(stderr, "option -t requires an argument\n");
exit(EXIT_FAILURE);
}
}
// For import directory
else if (strcmp(argv[i], "-D") == 0) {
if (i < argc - 1) {
if (wordexp(argv[++i], &full_path, 0) != 0) {
fprintf(stderr, "Invalid path %s\n", argv[i]);
exit(EXIT_FAILURE);
}
strcpy(arguments->dir, full_path.we_wordv[0]);
wordfree(&full_path);
} else {
fprintf(stderr, "option -D requires an argument\n");
exit(EXIT_FAILURE);
}
}
// For time zone
else if (strcmp(argv[i], "-T") == 0) {
if (i < argc - 1) {
arguments->threadNum = atoi(argv[++i]);
} else {
fprintf(stderr, "option -T requires an argument\n");
exit(EXIT_FAILURE);
}
}
// For temperory command TODO
else if (strcmp(argv[i], "--help") == 0) {
printHelp();
exit(EXIT_FAILURE);
} else {
fprintf(stderr, "wrong options\n");
printHelp();
exit(EXIT_FAILURE);
}
}
}
int32_t shellReadCommand(TAOS *con, char *command) {
unsigned hist_counter = history.hend;
char utf8_array[10] = "\0";
Command cmd;
memset(&cmd, 0, sizeof(cmd));
cmd.buffer = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE);
cmd.command = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE);
showOnScreen(&cmd);
// Read input.
char c;
while (1) {
c = getchar();
if (c < 0) { // For UTF-8
int count = countPrefixOnes(c);
utf8_array[0] = c;
for (int k = 1; k < count; k++) {
c = getchar();
utf8_array[k] = c;
}
insertChar(&cmd, utf8_array, count);
} else if (c < '\033') {
// Ctrl keys. TODO: Implement ctrl combinations
switch (c) {
case 1: // ctrl A
positionCursorHome(&cmd);
break;
case 3:
printf("\n");
resetCommand(&cmd, "");
kill(0, SIGINT);
break;
case 4: // EOF or Ctrl+D
printf("\n");
taos_close(con);
// write the history
write_history();
exitShell();
break;
case 5: // ctrl E
positionCursorEnd(&cmd);
break;
case 8:
backspaceChar(&cmd);
break;
case '\n':
case '\r':
printf("\n");
if (isReadyGo(&cmd)) {
sprintf(command, "%s%s", cmd.buffer, cmd.command);
taosMemoryFreeClear(cmd.buffer);
taosMemoryFreeClear(cmd.command);
return 0;
} else {
updateBuffer(&cmd);
}
break;
case 11: // Ctrl + K;
clearLineAfter(&cmd);
break;
case 12: // Ctrl + L;
system("clear");
showOnScreen(&cmd);
break;
case 21: // Ctrl + U
clearLineBefore(&cmd);
break;
}
} else if (c == '\033') {
c = getchar();
switch (c) {
case '[':
c = getchar();
switch (c) {
case 'A': // Up arrow
if (hist_counter != history.hstart) {
hist_counter = (hist_counter + MAX_HISTORY_SIZE - 1) % MAX_HISTORY_SIZE;
resetCommand(&cmd, (history.hist[hist_counter] == NULL) ? "" : history.hist[hist_counter]);
}
break;
case 'B': // Down arrow
if (hist_counter != history.hend) {
int next_hist = (hist_counter + 1) % MAX_HISTORY_SIZE;
if (next_hist != history.hend) {
resetCommand(&cmd, (history.hist[next_hist] == NULL) ? "" : history.hist[next_hist]);
} else {
resetCommand(&cmd, "");
}
hist_counter = next_hist;
}
break;
case 'C': // Right arrow
moveCursorRight(&cmd);
break;
case 'D': // Left arrow
moveCursorLeft(&cmd);
break;
case '1':
if ((c = getchar()) == '~') {
// Home key
positionCursorHome(&cmd);
}
break;
case '2':
if ((c = getchar()) == '~') {
// Insert key
}
break;
case '3':
if ((c = getchar()) == '~') {
// Delete key
deleteChar(&cmd);
}
break;
case '4':
if ((c = getchar()) == '~') {
// End key
positionCursorEnd(&cmd);
}
break;
case '5':
if ((c = getchar()) == '~') {
// Page up key
}
break;
case '6':
if ((c = getchar()) == '~') {
// Page down key
}
break;
case 72:
// Home key
positionCursorHome(&cmd);
break;
case 70:
// End key
positionCursorEnd(&cmd);
break;
}
break;
}
} else if (c == 0x7f) {
// press delete key
backspaceChar(&cmd);
} else {
insertChar(&cmd, &c, 1);
}
}
return 0;
}
void *shellLoopQuery(void *arg) {
if (indicator) {
getOldTerminalMode();
indicator = 0;
}
TAOS *con = (TAOS *)arg;
setThreadName("shellLoopQuery");
taosThreadCleanupPush(cleanup_handler, NULL);
char *command = taosMemoryMalloc(MAX_COMMAND_SIZE);
if (command == NULL){
tscError("failed to malloc command");
return NULL;
}
int32_t err = 0;
do {
// Read command from shell.
memset(command, 0, MAX_COMMAND_SIZE);
setTerminalMode();
err = shellReadCommand(con, command);
if (err) {
break;
}
resetTerminalMode();
} while (shellRunCommand(con, command) == 0);
taosMemoryFreeClear(command);
exitShell();
taosThreadCleanupPop(1);
return NULL;
}
void get_history_path(char *history) { sprintf(history, "%s/%s", getpwuid(getuid())->pw_dir, HISTORY_FILE); }
void clearScreen(int ecmd_pos, int cursor_pos) {
struct winsize w;
if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) {
//fprintf(stderr, "No stream device, and use default value(col 120, row 30)\n");
w.ws_col = 120;
w.ws_row = 30;
}
int cursor_x = cursor_pos / w.ws_col;
int cursor_y = cursor_pos % w.ws_col;
int command_x = ecmd_pos / w.ws_col;
positionCursor(cursor_y, LEFT);
positionCursor(command_x - cursor_x, DOWN);
fprintf(stdout, "\033[2K");
for (int i = 0; i < command_x; i++) {
positionCursor(1, UP);
fprintf(stdout, "\033[2K");
}
fflush(stdout);
}
void showOnScreen(Command *cmd) {
struct winsize w;
if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) {
//fprintf(stderr, "No stream device\n");
w.ws_col = 120;
w.ws_row = 30;
}
TdWchar wc;
int size = 0;
// Print out the command.
char *total_string = taosMemoryMalloc(MAX_COMMAND_SIZE);
memset(total_string, '\0', MAX_COMMAND_SIZE);
if (strcmp(cmd->buffer, "") == 0) {
sprintf(total_string, "%s%s", PROMPT_HEADER, cmd->command);
} else {
sprintf(total_string, "%s%s", CONTINUE_PROMPT, cmd->command);
}
int remain_column = w.ws_col;
/* size = cmd->commandSize + prompt_size; */
for (char *str = total_string; size < cmd->commandSize + prompt_size;) {
int ret = taosMbToWchar(&wc, str, MB_CUR_MAX);
if (ret < 0) break;
size += ret;
/* assert(size >= 0); */
int width = taosWcharWidth(wc);
if (remain_column > width) {
printf("%lc", wc);
remain_column -= width;
} else {
if (remain_column == width) {
printf("%lc\n\r", wc);
remain_column = w.ws_col;
} else {
printf("\n\r%lc", wc);
remain_column = w.ws_col - width;
}
}
str = total_string + size;
}
taosMemoryFree(total_string);
/* for (int i = 0; i < size; i++){ */
/* char c = total_string[i]; */
/* if (k % w.ws_col == 0) { */
/* printf("%c\n\r", c); */
/* } */
/* else { */
/* printf("%c", c); */
/* } */
/* k += 1; */
/* } */
// Position the cursor
int cursor_pos = cmd->screenOffset + prompt_size;
int ecmd_pos = cmd->endOffset + prompt_size;
int cursor_x = cursor_pos / w.ws_col;
int cursor_y = cursor_pos % w.ws_col;
// int cursor_y = cursor % w.ws_col;
int command_x = ecmd_pos / w.ws_col;
int command_y = ecmd_pos % w.ws_col;
// int command_y = (command.size() + prompt_size) % w.ws_col;
positionCursor(command_y, LEFT);
positionCursor(command_x, UP);
positionCursor(cursor_x, DOWN);
positionCursor(cursor_y, RIGHT);
fflush(stdout);
}
void cleanup_handler(void *arg) { resetTerminalMode(); }
void exitShell() {
resetTerminalMode();
exit(EXIT_SUCCESS);
}

279
tools/shell/src/backup/shellImport.c
View File

@ -1,279 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _GNU_SOURCE
#define _XOPEN_SOURCE
#define _DEFAULT_SOURCE
#include "os.h"
#include "shell.h"
#include "shellCommand.h"
#include "tglobal.h"
#include "tutil.h"
static char **shellSQLFiles = NULL;
static int32_t shellSQLFileNum = 0;
static char shellTablesSQLFile[TSDB_FILENAME_LEN] = {0};
typedef struct {
TdThread threadID;
int threadIndex;
int totalThreads;
void *taos;
} ShellThreadObj;
static int shellGetFilesNum(const char *directoryName, const char *prefix)
{
char cmd[1024] = { 0 };
sprintf(cmd, "ls %s/*.%s | wc -l ", directoryName, prefix);
char buf[1024] = { 0 };
if (taosSystem(cmd, buf, sizeof(buf)) < 0) {
fprintf(stderr, "ERROR: failed to execute:%s, error:%s\n", cmd, strerror(errno));
exit(0);
}
int fileNum = 0;
if (sscanf(buf, "%d", &fileNum) != 1) {
fprintf(stderr, "ERROR: failed to execute:%s, parse result error\n", cmd);
exit(0);
}
if (fileNum <= 0) {
fprintf(stderr, "ERROR: directory:%s is empry\n", directoryName);
exit(0);
}
return fileNum;
}
static void shellParseDirectory(const char *directoryName, const char *prefix, char **fileArray, int totalFiles)
{
char cmd[1024] = { 0 };
sprintf(cmd, "ls %s/*.%s | sort", directoryName, prefix);
char buf[1024] = { 0 };
if (taosSystem(cmd, buf, sizeof(buf)) < 0) {
fprintf(stderr, "ERROR: failed to execute:%s, error:%s\n", cmd, strerror(errno));
exit(0);
}
int fileNum = 0;
while (sscanf(buf, "%128s", fileArray[fileNum++])) {
if (strcmp(fileArray[fileNum-1], shellTablesSQLFile) == 0) {
fileNum--;
}
if (fileNum >= totalFiles) {
break;
}
}
if (fileNum != totalFiles) {
fprintf(stderr, "ERROR: directory:%s changed while read\n", directoryName);
exit(0);
}
}
static void shellCheckTablesSQLFile(const char *directoryName)
{
sprintf(shellTablesSQLFile, "%s/tables.sql", directoryName);
if (taosFStatFile(shellTablesSQLFile, NULL, NULL) < 0) {
shellTablesSQLFile[0] = 0;
}
}
static void shellMallocSQLFiles()
{
shellSQLFiles = (char**)taosMemoryCalloc(shellSQLFileNum, sizeof(char*));
for (int i = 0; i < shellSQLFileNum; i++) {
shellSQLFiles[i] = taosMemoryCalloc(1, TSDB_FILENAME_LEN);
}
}
static void shellGetDirectoryFileList(char *inputDir)
{
if (!taosDirExist(inputDir)) {
fprintf(stderr, "ERROR: %s not exist\n", inputDir);
exit(0);
}
if (taosIsDir(inputDir)) {
shellCheckTablesSQLFile(inputDir);
shellSQLFileNum = shellGetFilesNum(inputDir, "sql");
int totalSQLFileNum = shellSQLFileNum;
if (shellTablesSQLFile[0] != 0) {
shellSQLFileNum--;
}
shellMallocSQLFiles();
shellParseDirectory(inputDir, "sql", shellSQLFiles, shellSQLFileNum);
fprintf(stdout, "\nstart to dispose %d files in %s\n", totalSQLFileNum, inputDir);
}
else {
fprintf(stderr, "ERROR: %s is not a directory\n", inputDir);
exit(0);
}
}
static void shellSourceFile(TAOS *con, char *fptr) {
wordexp_t full_path;
int read_len = 0;
char * cmd = taosMemoryMalloc(tsMaxSQLStringLen);
size_t cmd_len = 0;
char * line = NULL;
if (wordexp(fptr, &full_path, 0) != 0) {
fprintf(stderr, "ERROR: illegal file name\n");
taosMemoryFree(cmd);
return;
}
char *fname = full_path.we_wordv[0];
if (fname == NULL) {
fprintf(stderr, "ERROR: invalid filename\n");
taosMemoryFree(cmd);
return;
}
/*
if (access(fname, F_OK) != 0) {
fprintf(stderr, "ERROR: file %s is not exist\n", fptr);
wordfree(&full_path);
taosMemoryFree(cmd);
return;
}
if (access(fname, R_OK) != 0) {
fprintf(stderr, "ERROR: file %s is not readable\n", fptr);
wordfree(&full_path);
taosMemoryFree(cmd);
return;
}
*/
// FILE *f = fopen(fname, "r");
TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ | TD_FILE_STREAM);
if (pFile == NULL) {
fprintf(stderr, "ERROR: failed to open file %s\n", fname);
wordfree(&full_path);
taosMemoryFree(cmd);
return;
}
fprintf(stdout, "begin import file:%s\n", fname);
int lineNo = 0;
while ((read_len = taosGetLineFile(pFile, &line)) != -1) {
++lineNo;
if (read_len >= tsMaxSQLStringLen) continue;
line[--read_len] = '\0';
if (read_len == 0 || isCommentLine(line)) { // line starts with #
continue;
}
if (line[read_len - 1] == '\\') {
line[read_len - 1] = ' ';
memcpy(cmd + cmd_len, line, read_len);
cmd_len += read_len;
continue;
}
memcpy(cmd + cmd_len, line, read_len);
TAOS_RES* pSql = taos_query(con, cmd);
int32_t code = taos_errno(pSql);
if (code != 0) {
fprintf(stderr, "DB error: %s: %s (%d)\n", taos_errstr(pSql), fname, lineNo);
}
/* free local resouce: allocated memory/metric-meta refcnt */
taos_free_result(pSql);
memset(cmd, 0, MAX_COMMAND_SIZE);
cmd_len = 0;
}
taosMemoryFree(cmd);
if(line != NULL) taosMemoryFree(line);
wordfree(&full_path);
taosCloseFile(&pFile);
}
void* shellImportThreadFp(void *arg)
{
ShellThreadObj *pThread = (ShellThreadObj*)arg;
setThreadName("shellImportThrd");
for (int f = 0; f < shellSQLFileNum; ++f) {
if (f % pThread->totalThreads == pThread->threadIndex) {
char *SQLFileName = shellSQLFiles[f];
shellSourceFile(pThread->taos, SQLFileName);
}
}
return NULL;
}
static void shellRunImportThreads(SShellArguments* _args)
{
TdThreadAttr thattr;
ShellThreadObj *threadObj = (ShellThreadObj *)taosMemoryCalloc(_args->threadNum, sizeof(ShellThreadObj));
for (int t = 0; t < _args->threadNum; ++t) {
ShellThreadObj *pThread = threadObj + t;
pThread->threadIndex = t;
pThread->totalThreads = _args->threadNum;
pThread->taos = taos_connect(_args->host, _args->user, _args->password, _args->database, tsDnodeShellPort);
if (pThread->taos == NULL) {
fprintf(stderr, "ERROR: thread:%d failed connect to TDengine, error:%s\n", pThread->threadIndex, "null taos"/*taos_errstr(pThread->taos)*/);
exit(0);
}
taosThreadAttrInit(&thattr);
taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&(pThread->threadID), &thattr, shellImportThreadFp, (void*)pThread) != 0) {
fprintf(stderr, "ERROR: thread:%d failed to start\n", pThread->threadIndex);
exit(0);
}
}
for (int t = 0; t < _args->threadNum; ++t) {
taosThreadJoin(threadObj[t].threadID, NULL);
}
for (int t = 0; t < _args->threadNum; ++t) {
taos_close(threadObj[t].taos);
}
taosMemoryFree(threadObj);
}
void source_dir(TAOS* con, SShellArguments* _args) {
shellGetDirectoryFileList(_args->dir);
int64_t start = taosGetTimestampMs();
if (shellTablesSQLFile[0] != 0) {
shellSourceFile(con, shellTablesSQLFile);
int64_t end = taosGetTimestampMs();
fprintf(stdout, "import %s finished, time spent %.2f seconds\n", shellTablesSQLFile, (end - start) / 1000.0);
}
shellRunImportThreads(_args);
int64_t end = taosGetTimestampMs();
fprintf(stdout, "import %s finished, time spent %.2f seconds\n", _args->dir, (end - start) / 1000.0);
}

325
tools/shell/src/backup/shellWindows.c
View File

@ -1,325 +0,0 @@
/*******************************************************************
* Copyright (c) 2017 by TAOS Technologies, Inc.
* All rights reserved.
*
* This file is proprietary and confidential to TAOS Technologies.
* No part of this file may be reproduced, stored, transmitted,
* disclosed or used in any form or by any means other than as
* expressly provided by the written permission from Jianhui Tao
*
* ****************************************************************/
#include <assert.h>
#include <regex.h>
#include <stdio.h>
#include "../../../../include/client/taos.h"
#include "os.h"
#include "shell.h"
#include "shellCommand.h"
extern char configDir[];
char WINCLIENT_VERSION[] = "Welcome to the TDengine shell from %s, Client Version:%s\n"
"Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n";
void printVersion() {
printf("version: %s\n", version);
}
void printHelp() {
char indent[10] = " ";
printf("taos shell is used to test the TDengine database\n");
printf("%s%s\n", indent, "-h");
printf("%s%s%s\n", indent, indent, "TDengine server FQDN to connect. The default host is localhost.");
printf("%s%s\n", indent, "-p");
printf("%s%s%s\n", indent, indent, "The password to use when connecting to the server.");
printf("%s%s\n", indent, "-P");
printf("%s%s%s\n", indent, indent, "The TCP/IP port number to use for the connection");
printf("%s%s\n", indent, "-u");
printf("%s%s%s\n", indent, indent, "The user name to use when connecting to the server.");
printf("%s%s\n", indent, "-A");
printf("%s%s%s\n", indent, indent, "The user auth to use when connecting to the server.");
printf("%s%s\n", indent, "-c");
printf("%s%s%s\n", indent, indent, "Configuration directory.");
printf("%s%s\n", indent, "-C");
printf("%s%s%s\n", indent, indent, "Dump configuration.");
printf("%s%s\n", indent, "-s");
printf("%s%s%s\n", indent, indent, "Commands to run without enter the shell.");
printf("%s%s\n", indent, "-r");
printf("%s%s%s\n", indent, indent, "Output time as unsigned long..");
printf("%s%s\n", indent, "-f");
printf("%s%s%s\n", indent, indent, "Script to run without enter the shell.");
printf("%s%s\n", indent, "-d");
printf("%s%s%s\n", indent, indent, "Database to use when connecting to the server.");
printf("%s%s\n", indent, "-t");
printf("%s%s%s\n", indent, indent, "Time zone of the shell, default is local.");
printf("%s%s\n", indent, "-n");
printf("%s%s%s\n", indent, indent, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup|sync|speed|fqdn.");
printf("%s%s\n", indent, "-l");
printf("%s%s%s\n", indent, indent, "Packet length used for net test, default is 1000 bytes.");
printf("%s%s\n", indent, "-N");
printf("%s%s%s\n", indent, indent, "Packet numbers used for net test, default is 100.");
printf("%s%s\n", indent, "-S");
printf("%s%s%s\n", indent, indent, "Packet type used for net test, default is TCP.");
printf("%s%s\n", indent, "-V");
printf("%s%s%s\n", indent, indent, "Print program version.");
exit(EXIT_SUCCESS);
}
char g_password[SHELL_MAX_PASSWORD_LEN];
void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) {
for (int i = 1; i < argc; i++) {
// for host
if (strcmp(argv[i], "-h") == 0) {
if (i < argc - 1) {
arguments->host = argv[++i];
} else {
fprintf(stderr, "option -h requires an argument\n");
exit(EXIT_FAILURE);
}
}
// for password
else if ((strncmp(argv[i], "-p", 2) == 0)
|| (strncmp(argv[i], "--password", 10) == 0)) {
arguments->is_use_passwd = true;
strcpy(tsOsName, "Windows");
printf(WINCLIENT_VERSION, tsOsName, taos_get_client_info());
if ((strlen(argv[i]) == 2)
|| (strncmp(argv[i], "--password", 10) == 0)) {
printf("Enter password: ");
taosSetConsoleEcho(false);
if (scanf("%s", g_password) > 1) {
fprintf(stderr, "password read error!\n");
}
taosSetConsoleEcho(true);
getchar();
} else {
tstrncpy(g_password, (char *)(argv[i] + 2), SHELL_MAX_PASSWORD_LEN);
}
arguments->password = g_password;
strcpy(argv[i], "");
argc -= 1;
}
// for management port
else if (strcmp(argv[i], "-P") == 0) {
if (i < argc - 1) {
arguments->port = atoi(argv[++i]);
} else {
fprintf(stderr, "option -P requires an argument\n");
exit(EXIT_FAILURE);
}
}
// for user
else if (strcmp(argv[i], "-u") == 0) {
if (i < argc - 1) {
arguments->user = argv[++i];
} else {
fprintf(stderr, "option -u requires an argument\n");
exit(EXIT_FAILURE);
}
} else if (strcmp(argv[i], "-A") == 0) {
if (i < argc - 1) {
arguments->auth = argv[++i];
} else {
fprintf(stderr, "option -A requires an argument\n");
exit(EXIT_FAILURE);
}
} else if (strcmp(argv[i], "-c") == 0) {
if (i < argc - 1) {
char *tmp = argv[++i];
if (strlen(tmp) >= TSDB_FILENAME_LEN) {
fprintf(stderr, "config file path: %s overflow max len %d\n", tmp, TSDB_FILENAME_LEN - 1);
exit(EXIT_FAILURE);
}
strcpy(configDir, tmp);
} else {
fprintf(stderr, "Option -c requires an argument\n");
exit(EXIT_FAILURE);
}
} else if (strcmp(argv[i], "-C") == 0) {
arguments->dump_config = true;
} else if (strcmp(argv[i], "-s") == 0) {
if (i < argc - 1) {
arguments->commands = argv[++i];
} else {
fprintf(stderr, "option -s requires an argument\n");
exit(EXIT_FAILURE);
}
} else if (strcmp(argv[i], "-r") == 0) {
arguments->is_raw_time = true;
}
// For temperory batch commands to run TODO
else if (strcmp(argv[i], "-f") == 0) {
if (i < argc - 1) {
strcpy(arguments->file, argv[++i]);
} else {
fprintf(stderr, "option -f requires an argument\n");
exit(EXIT_FAILURE);
}
}
// for default database
else if (strcmp(argv[i], "-d") == 0) {
if (i < argc - 1) {
arguments->database = argv[++i];
} else {
fprintf(stderr, "option -d requires an argument\n");
exit(EXIT_FAILURE);
}
}
// For time zone
else if (strcmp(argv[i], "-t") == 0) {
if (i < argc - 1) {
arguments->timezone = argv[++i];
} else {
fprintf(stderr, "option -t requires an argument\n");
exit(EXIT_FAILURE);
}
}
else if (strcmp(argv[i], "-n") == 0) {
if (i < argc - 1) {
arguments->netTestRole = argv[++i];
} else {
fprintf(stderr, "option -n requires an argument\n");
exit(EXIT_FAILURE);
}
}
else if (strcmp(argv[i], "-l") == 0) {
if (i < argc - 1) {
arguments->pktLen = atoi(argv[++i]);
} else {
fprintf(stderr, "option -l requires an argument\n");
exit(EXIT_FAILURE);
}
}
else if (strcmp(argv[i], "-N") == 0) {
if (i < argc - 1) {
arguments->pktNum = atoi(argv[++i]);
} else {
fprintf(stderr, "option -N requires an argument\n");
exit(EXIT_FAILURE);
}
}
else if (strcmp(argv[i], "-S") == 0) {
if (i < argc - 1) {
arguments->pktType = argv[++i];
} else {
fprintf(stderr, "option -S requires an argument\n");
exit(EXIT_FAILURE);
}
}
else if (strcmp(argv[i], "-V") == 0) {
printVersion();
exit(EXIT_SUCCESS);
}
// For temperory command TODO
else if (strcmp(argv[i], "--help") == 0) {
printHelp();
exit(EXIT_SUCCESS);
} else {
fprintf(stderr, "wrong options\n");
printHelp();
exit(EXIT_FAILURE);
}
}
}
void shellPrintContinuePrompt() { printf("%s", CONTINUE_PROMPT); }
void shellPrintPrompt() { printf("%s", PROMPT_HEADER); }
void updateBuffer(Command *cmd) {
if (regex_match(cmd->buffer, "(\\s+$)|(^$)", REG_EXTENDED)) strcat(cmd->command, " ");
strcat(cmd->buffer, cmd->command);
memset(cmd->command, 0, MAX_COMMAND_SIZE);
cmd->cursorOffset = 0;
}
int isReadyGo(Command *cmd) {
char *total = taosMemoryMalloc(MAX_COMMAND_SIZE);
memset(total, 0, MAX_COMMAND_SIZE);
sprintf(total, "%s%s", cmd->buffer, cmd->command);
char *reg_str =
"(^.*;\\s*$)|(^\\s*$)|(^\\s*exit\\s*$)|(^\\s*q\\s*$)|(^\\s*quit\\s*$)|(^"
"\\s*clear\\s*$)";
if (regex_match(total, reg_str, REG_EXTENDED | REG_ICASE)) {
taosMemoryFree(total);
return 1;
}
taosMemoryFree(total);
return 0;
}
void insertChar(Command *cmd, char c) {
// TODO: Check if the length enough.
if (cmd->cursorOffset >= MAX_COMMAND_SIZE) {
fprintf(stdout, "sql is larger than %d bytes", MAX_COMMAND_SIZE);
return;
}
cmd->command[cmd->cursorOffset++] = c;
}
int32_t shellReadCommand(TAOS *con, char command[]) {
Command cmd;
memset(&cmd, 0, sizeof(cmd));
cmd.buffer = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE);
cmd.command = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE);
// Read input.
char c;
while (1) {
c = getchar();
switch (c) {
case '\n':
case '\r':
if (isReadyGo(&cmd)) {
sprintf(command, "%s%s", cmd.buffer, cmd.command);
taosMemoryFree(cmd.buffer);
cmd.buffer = NULL;
taosMemoryFree(cmd.command);
cmd.command = NULL;
return 0;
} else {
shellPrintContinuePrompt();
updateBuffer(&cmd);
}
break;
default:
insertChar(&cmd, c);
}
}
return 0;
}
void *shellLoopQuery(void *arg) {
TAOS *con = (TAOS *)arg;
char *command = taosMemoryMalloc(MAX_COMMAND_SIZE);
if (command == NULL) return NULL;
int32_t err = 0;
do {
memset(command, 0, MAX_COMMAND_SIZE);
shellPrintPrompt();
// Read command from shell.
err = shellReadCommand(con, command);
if (err) {
break;
}
} while (shellRunCommand(con, command) == 0);
return NULL;
}
void get_history_path(char *history) { sprintf(history, "C:/TDengine/%s", HISTORY_FILE); }
void exitShell() { exit(EXIT_SUCCESS); }

354
tools/shell/src/shellArguments.c Normal file
View File

@ -0,0 +1,354 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "shellInt.h"
#define SHELL_HOST "The auth string to use when connecting to the server."
#define SHELL_PORT "The TCP/IP port number to use for the connection."
#define SHELL_USER "The user name to use when connecting to the server."
#define SHELL_PASSWORD "The password to use when connecting to the server."
#define SHELL_AUTH "The auth string to use when connecting to the server."
#define SHELL_GEN_AUTH "Generate auth string from password."
#define SHELL_CFG_DIR "Configuration directory."
#define SHELL_DMP_CFG "Dump configuration."
#define SHELL_CMD "Commands to run without enter the shell."
#define SHELL_RAW_TIME "Output time as uint64_t."
#define SHELL_FILE "Script to run without enter the shell."
#define SHELL_DB "Database to use when connecting to the server."
#define SHELL_CHECK "Check the service status."
#define SHELL_STARTUP "Check the details of the service status."
#define SHELL_WIDTH "Set the default binary display width, default is 30."
#define SHELL_NET_ROLE "Net role when network connectivity test, options: client|server."
#define SHELL_PKG_LEN "Packet length used for net test, default is 1024 bytes."
#define SHELL_PKT_NUM "Packet numbers used for net test, default is 100."
#define SHELL_VERSION "Print program version."
#define SHELL_EMAIL "<support@taosdata.com>"
void shellPrintHelp() {
char indent[] = " ";
printf("Usage: taos [OPTION...] \n\n");
printf("%s%s%s%s\n", indent, "-a,", indent, SHELL_AUTH);
printf("%s%s%s%s\n", indent, "-A,", indent, SHELL_GEN_AUTH);
printf("%s%s%s%s\n", indent, "-c,", indent, SHELL_CFG_DIR);
printf("%s%s%s%s\n", indent, "-C,", indent, SHELL_DMP_CFG);
printf("%s%s%s%s\n", indent, "-d,", indent, SHELL_DB);
printf("%s%s%s%s\n", indent, "-f,", indent, SHELL_FILE);
printf("%s%s%s%s\n", indent, "-h,", indent, SHELL_HOST);
printf("%s%s%s%s\n", indent, "-k,", indent, SHELL_CHECK);
printf("%s%s%s%s\n", indent, "-l,", indent, SHELL_PKG_LEN);
printf("%s%s%s%s\n", indent, "-n,", indent, SHELL_NET_ROLE);
printf("%s%s%s%s\n", indent, "-N,", indent, SHELL_PKT_NUM);
printf("%s%s%s%s\n", indent, "-p,", indent, SHELL_PASSWORD);
printf("%s%s%s%s\n", indent, "-P,", indent, SHELL_PORT);
printf("%s%s%s%s\n", indent, "-r,", indent, SHELL_RAW_TIME);
printf("%s%s%s%s\n", indent, "-s,", indent, SHELL_CMD);
printf("%s%s%s%s\n", indent, "-t,", indent, SHELL_STARTUP);
printf("%s%s%s%s\n", indent, "-u,", indent, SHELL_USER);
printf("%s%s%s%s\n", indent, "-w,", indent, SHELL_WIDTH);
printf("%s%s%s%s\n", indent, "-V,", indent, SHELL_VERSION);
printf("\n\nReport bugs to %s.\n", SHELL_EMAIL);
}
static int32_t shellParseSingleOpt(int32_t key, char *arg) {
SShellArgs *pArgs = &shell.args;
switch (key) {
case 'h':
pArgs->host = arg;
break;
case 'P':
pArgs->port = atoi(arg);
break;
case 'u':
pArgs->user = arg;
break;
case 'p':
break;
case 'a':
pArgs->auth = arg;
break;
case 'A':
pArgs->is_gen_auth = true;
break;
case 'c':
pArgs->cfgdir = arg;
break;
case 'C':
pArgs->is_dump_config = true;
break;
case 's':
pArgs->commands = arg;
break;
case 'r':
pArgs->is_raw_time = true;
break;
case 'f':
tstrncpy(pArgs->file, arg, sizeof(pArgs->file));
break;
case 'd':
pArgs->database = arg;
break;
case 'k':
pArgs->is_check = true;
break;
case 't':
pArgs->is_startup = true;
break;
case 'w':
pArgs->displayWidth = atoi(arg);
break;
case 'n':
pArgs->netrole = arg;
break;
case 'l':
pArgs->pktLen = atoi(arg);
break;
case 'N':
pArgs->pktNum = atoi(arg);
break;
case 'V':
pArgs->is_version = true;
break;
case '?':
pArgs->is_help = true;
break;
case 1:
pArgs->abort = 1;
break;
default:
return ARGP_ERR_UNKNOWN;
}
return 0;
}
int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) {
SShellArgs *pArgs = &shell.args;
for (int i = 1; i < argc; i++) {
if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "--usage") == 0 || strcmp(argv[i], "-?") == 0) {
shellParseSingleOpt('?', NULL);
return 0;
}
char *key = argv[i];
int32_t keyLen = strlen(key);
if (keyLen != 2) {
fprintf(stderr, "invalid option %s\n", key);
return -1;
}
if (key[0] != '-') {
fprintf(stderr, "invalid option %s\n", key);
return -1;
}
if (key[1] == 'h' || key[1] == 'P' || key[1] == 'u' || key[1] == 'a' || key[1] == 'c' || key[1] == 's' ||
key[1] == 'f' || key[1] == 'd' || key[1] == 'w' || key[1] == 'n' || key[1] == 'l' || key[1] == 'N') {
if (i + 1 >= argc) {
fprintf(stderr, "option %s requires an argument\n", key);
return -1;
}
char *val = argv[i + 1];
if (val[0] == '-') {
fprintf(stderr, "option %s requires an argument\n", key);
return -1;
}
shellParseSingleOpt(key[1], val);
i++;
} else if (key[1] == 'p' || key[1] == 'A' || key[1] == 'c' || key[1] == 'r' || key[1] == 'k' || key[1] == 't' ||
key[1] == 'V') {
shellParseSingleOpt(key[1], NULL);
} else {
fprintf(stderr, "invalid option %s\n", key);
return -1;
}
}
return 0;
}
#ifdef LINUX
#include <argp.h>
#include <termio.h>
const char *argp_program_version = version;
const char *argp_program_bug_address = SHELL_EMAIL;
static struct argp_option shellOptions[] = {
{"host", 'h', "HOST", 0, SHELL_HOST},
{"port", 'P', "PORT", 0, SHELL_PORT},
{"user", 'u', "USER", 0, SHELL_USER},
{0, 'p', 0, 0, SHELL_PASSWORD},
{"auth", 'a', "AUTH", 0, SHELL_AUTH},
{"generate-auth", 'A', 0, 0, SHELL_GEN_AUTH},
{"config-dir", 'c', "DIR", 0, SHELL_CFG_DIR},
{"dump-config", 'C', 0, 0, SHELL_DMP_CFG},
{"commands", 's', "COMMANDS", 0, SHELL_CMD},
{"raw-time", 'r', 0, 0, SHELL_RAW_TIME},
{"file", 'f', "FILE", 0, SHELL_FILE},
{"database", 'd', "DATABASE", 0, SHELL_DB},
{"check", 'k', 0, 0, SHELL_CHECK},
{"startup", 't', 0, 0, SHELL_STARTUP},
{"display-width", 'w', "WIDTH", 0, SHELL_WIDTH},
{"netrole", 'n', "NETROLE", 0, SHELL_NET_ROLE},
{"pktlen", 'l', "PKTLEN", 0, SHELL_PKG_LEN},
{"pktnum", 'N', "PKTNUM", 0, SHELL_PKT_NUM},
{0},
};
static error_t shellParseOpt(int32_t key, char *arg, struct argp_state *state) { return shellParseSingleOpt(key, arg); }
static struct argp shellArgp = {shellOptions, shellParseOpt, "", ""};
static void shellParseArgsUseArgp(int argc, char *argv[]) {
argp_program_version = shell.info.programVersion;
argp_parse(&shellArgp, argc, argv, 0, 0, &shell.args);
}
#endif
static void shellInitArgs(int argc, char *argv[]) {
for (int i = 1; i < argc; i++) {
if (strncmp(argv[i], "-p", 2) == 0) {
printf(shell.info.clientVersion, tsOsName, taos_get_client_info());
if (strlen(argv[i]) == 2) {
printf("Enter password: ");
taosSetConsoleEcho(false);
if (scanf("%20s", shell.args.password) > 1) {
fprintf(stderr, "password reading error\n");
}
taosSetConsoleEcho(true);
if (EOF == getchar()) {
fprintf(stderr, "getchar() return EOF\n");
}
} else {
tstrncpy(shell.args.password, (char *)(argv[i] + 2), sizeof(shell.args.password));
strcpy(argv[i], "-p");
}
}
}
if (strlen(shell.args.password) == 0) {
tstrncpy(shell.args.password, TSDB_DEFAULT_PASS, sizeof(shell.args.password));
}
SShellArgs *pArgs = &shell.args;
pArgs->user = TSDB_DEFAULT_USER;
pArgs->pktLen = SHELL_DEF_PKG_LEN;
pArgs->pktNum = SHELL_DEF_PKG_NUM;
pArgs->displayWidth = SHELL_DEFAULT_MAX_BINARY_DISPLAY_WIDTH;
}
static int32_t shellCheckArgs() {
SShellArgs *pArgs = &shell.args;
if (pArgs->host != NULL && (strlen(pArgs->host) <= 0 || strlen(pArgs->host) > TSDB_FQDN_LEN)) {
printf("Invalid host:%s\n", pArgs->host);
return -1;
}
if (pArgs->user != NULL && (strlen(pArgs->user) <= 0 || strlen(pArgs->user) > TSDB_USER_LEN)) {
printf("Invalid user:%s\n", pArgs->user);
return -1;
}
if (pArgs->auth != NULL && (strlen(pArgs->auth) <= 0 || strlen(pArgs->auth) > TSDB_PASSWORD_LEN)) {
printf("Invalid auth:%s\n", pArgs->auth);
return -1;
}
if (pArgs->database != NULL && (strlen(pArgs->database) <= 0 || strlen(pArgs->database) > TSDB_DB_NAME_LEN)) {
printf("Invalid database:%s\n", pArgs->database);
return -1;
}
if (pArgs->file[0] != 0) {
char fullname[PATH_MAX] = {0};
if (taosExpandDir(pArgs->file, fullname, PATH_MAX) == 0) {
tstrncpy(pArgs->file, fullname, PATH_MAX);
}
}
if (pArgs->cfgdir != NULL) {
if (strlen(pArgs->cfgdir) <= 0 || strlen(pArgs->cfgdir) >= PATH_MAX) {
printf("Invalid cfgdir:%s\n", pArgs->cfgdir);
return -1;
} else {
if (taosExpandDir(pArgs->cfgdir, configDir, PATH_MAX) != 0) {
tstrncpy(configDir, pArgs->cfgdir, PATH_MAX);
}
}
}
if (pArgs->commands != NULL && (strlen(pArgs->commands) <= 0)) {
printf("Invalid commands:%s\n", pArgs->commands);
return -1;
}
if (pArgs->netrole != NULL && !(strcmp(pArgs->netrole, "client") == 0 || strcmp(pArgs->netrole, "server") == 0)) {
printf("Invalid netrole:%s\n", pArgs->netrole);
return -1;
}
if (pArgs->password != NULL && (strlen(pArgs->password) <= 0)) {
printf("Invalid password\n");
return -1;
}
if (pArgs->pktLen < SHELL_MIN_PKG_LEN || pArgs->pktLen > SHELL_MAX_PKG_LEN) {
printf("Invalid pktLen:%d, range:[%d, %d]\n", pArgs->pktLen, SHELL_MIN_PKG_LEN, SHELL_MAX_PKG_LEN);
return -1;
}
if (pArgs->pktNum < SHELL_MIN_PKG_NUM || pArgs->pktNum > SHELL_MAX_PKG_NUM) {
printf("Invalid pktNum:%d, range:[%d, %d]\n", pArgs->pktNum, SHELL_MIN_PKG_NUM, SHELL_MAX_PKG_NUM);
return -1;
}
if (pArgs->displayWidth <= 0 || pArgs->displayWidth > 10 * 1024) {
printf("Invalid displayWidth:%d, range:[1, 10 * 1024]\n", pArgs->displayWidth);
return -1;
}
return 0;
}
int32_t shellParseArgs(int32_t argc, char *argv[]) {
shellInitArgs(argc, argv);
shell.info.clientVersion =
"Welcome to the TDengine shell from %s, Client Version:%s\n"
"Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n";
shell.info.promptHeader = "taos> ";
shell.info.promptContinue = " -> ";
shell.info.promptSize = 6;
snprintf(shell.info.programVersion, sizeof(shell.info.programVersion), "version: %s", version);
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
shell.info.osname = "Windows";
snprintf(shell.history.file, TSDB_FILENAME_LEN, "C:/TDengine/%s", SHELL_HISTORY_FILE);
if (shellParseArgsWithoutArgp(argc, argv) != 0) return -1;
#elif defined(_TD_DARWIN_64)
shell.info.osname = "Darwin";
snprintf(shell.history.file, TSDB_FILENAME_LEN, "%s/%s", getpwuid(getuid())->pw_dir, SHELL_HISTORY_FILE);
if (shellParseArgsWithoutArgp(argc, argv) != 0) return -1;
#else
shell.info.osname = "Linux";
snprintf(shell.history.file, TSDB_FILENAME_LEN, "%s/%s", getenv("HOME"), SHELL_HISTORY_FILE);
shellParseArgsUseArgp(argc, argv);
// if (shellParseArgsWithoutArgp(argc, argv) != 0) return -1;
if (shell.args.abort) {
return -1;
}
#endif
return shellCheckArgs();
}

496
tools/shell/src/shellCommand.c
View File

@ -14,22 +14,122 @@
*/
#define __USE_XOPEN
#include "shellInt.h"
#include "shellCommand.h"
#include "os.h"
#include "shell.h"
#include <regex.h>
#define LEFT 1
#define RIGHT 2
#define UP 3
#define DOWN 4
#define PSIZE shell.info.promptSize
typedef struct {
char widthInString;
char widthOnScreen;
} UTFCodeInfo;
char *buffer;
char *command;
uint32_t commandSize;
uint32_t bufferSize;
uint32_t cursorOffset;
uint32_t screenOffset;
uint32_t endOffset;
} SShellCmd;
int countPrefixOnes(unsigned char c) {
unsigned char mask = 127;
static int32_t shellCountPrefixOnes(uint8_t c);
static void shellGetPrevCharSize(const char *str, int32_t pos, int32_t *size, int32_t *width);
static void shellGetNextCharSize(const char *str, int32_t pos, int32_t *size, int32_t *width);
static void shellInsertChar(SShellCmd *cmd, char *c, int size);
static void shellBackspaceChar(SShellCmd *cmd);
static void shellClearLineBefore(SShellCmd *cmd);
static void shellClearLineAfter(SShellCmd *cmd);
static void shellDeleteChar(SShellCmd *cmd);
static void shellMoveCursorLeft(SShellCmd *cmd);
static void shellMoveCursorRight(SShellCmd *cmd);
static void shellPositionCursorHome(SShellCmd *cmd);
static void shellPositionCursorEnd(SShellCmd *cmd);
static void shellPrintChar(char c, int32_t times);
static void shellPositionCursor(int32_t step, int32_t direction);
static void shellUpdateBuffer(SShellCmd *cmd);
static int32_t shellIsReadyGo(SShellCmd *cmd);
static void shellGetMbSizeInfo(const char *str, int32_t *size, int32_t *width);
static void shellResetCommand(SShellCmd *cmd, const char s[]);
static void shellClearScreen(int32_t ecmd_pos, int32_t cursor_pos);
static void shellShowOnScreen(SShellCmd *cmd);
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
static void shellPrintContinuePrompt() { printf("%s", shell.args.promptContinue); }
static void shellPrintPrompt() { printf("%s", shell.args.promptHeader); }
void shellUpdateBuffer(SShellCmd *cmd) {
if (shellRegexMatch(cmd->buffer, "(\\s+$)|(^$)", REG_EXTENDED)) strcat(cmd->command, " ");
strcat(cmd->buffer, cmd->command);
memset(cmd->command, 0, SHELL_MAX_COMMAND_SIZE);
cmd->cursorOffset = 0;
}
int shellIsReadyGo(SShellCmd *cmd) {
char *total = taosMemoryMalloc(SHELL_MAX_COMMAND_SIZE);
memset(total, 0, SHELL_MAX_COMMAND_SIZE);
sprintf(total, "%s%s", cmd->buffer, cmd->command);
char *reg_str =
"(^.*;\\s*$)|(^\\s*$)|(^\\s*exit\\s*$)|(^\\s*q\\s*$)|(^\\s*quit\\s*$)|(^"
"\\s*clear\\s*$)";
if (shellRegexMatch(total, reg_str, REG_EXTENDED | REG_ICASE)) {
taosMemoryFree(total);
return 1;
}
taosMemoryFree(total);
return 0;
}
void shellInsertChar(SShellCmd *cmd, char c) {
if (cmd->cursorOffset >= SHELL_MAX_COMMAND_SIZE) {
fprintf(stdout, "sql is larger than %d bytes", SHELL_MAX_COMMAND_SIZE);
return;
}
cmd->command[cmd->cursorOffset++] = c;
}
int32_t shellReadCommand(char command[]) {
SShellCmd cmd;
memset(&cmd, 0, sizeof(cmd));
cmd.buffer = (char *)taosMemoryCalloc(1, SHELL_MAX_COMMAND_SIZE);
cmd.command = (char *)taosMemoryCalloc(1, SHELL_MAX_COMMAND_SIZE);
// Read input.
char c;
while (1) {
c = getchar();
switch (c) {
case '\n':
case '\r':
if (shellIsReadyGo(&cmd)) {
sprintf(command, "%s%s", cmd.buffer, cmd.command);
taosMemoryFree(cmd.buffer);
cmd.buffer = NULL;
taosMemoryFree(cmd.command);
cmd.command = NULL;
return 0;
} else {
shellPrintContinuePrompt();
shellUpdateBuffer(&cmd);
}
break;
default:
shellInsertChar(&cmd, c);
}
}
return 0;
}
#else
int32_t shellCountPrefixOnes(uint8_t c) {
uint8_t mask = 127;
mask = ~mask;
int ret = 0;
int32_t ret = 0;
while ((c & mask) != 0) {
ret++;
c <<= 1;
@ -38,7 +138,7 @@ int countPrefixOnes(unsigned char c) {
return ret;
}
void getPrevCharSize(const char *str, int pos, int *size, int *width) {
void shellGetPrevCharSize(const char *str, int32_t pos, int32_t *size, int32_t *width) {
assert(pos > 0);
TdWchar wc;
@ -48,16 +148,16 @@ void getPrevCharSize(const char *str, int pos, int *size, int *width) {
while (--pos >= 0) {
*size += 1;
if (str[pos] > 0 || countPrefixOnes((unsigned char)str[pos]) > 1) break;
if (str[pos] > 0 || shellCountPrefixOnes((uint8_t)str[pos]) > 1) break;
}
int rc = taosMbToWchar(&wc, str + pos, MB_CUR_MAX);
int32_t rc = taosMbToWchar(&wc, str + pos, MB_CUR_MAX);
assert(rc == *size);
*width = taosWcharWidth(wc);
}
void getNextCharSize(const char *str, int pos, int *size, int *width) {
void shellGetNextCharSize(const char *str, int32_t pos, int32_t *size, int32_t *width) {
assert(pos >= 0);
TdWchar wc;
@ -65,13 +165,13 @@ void getNextCharSize(const char *str, int pos, int *size, int *width) {
*width = taosWcharWidth(wc);
}
void insertChar(Command *cmd, char *c, int size) {
void shellInsertChar(SShellCmd *cmd, char *c, int32_t size) {
assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset);
TdWchar wc;
if (taosMbToWchar(&wc, c, size) < 0) return;
clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size);
shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE);
/* update the buffer */
memmove(cmd->command + cmd->cursorOffset + size, cmd->command + cmd->cursorOffset,
cmd->commandSize - cmd->cursorOffset);
@ -81,122 +181,122 @@ void insertChar(Command *cmd, char *c, int size) {
cmd->cursorOffset += size;
cmd->screenOffset += taosWcharWidth(wc);
cmd->endOffset += taosWcharWidth(wc);
showOnScreen(cmd);
shellShowOnScreen(cmd);
}
void backspaceChar(Command *cmd) {
void shellBackspaceChar(SShellCmd *cmd) {
assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset);
if (cmd->cursorOffset > 0) {
clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size);
int size = 0;
int width = 0;
getPrevCharSize(cmd->command, cmd->cursorOffset, &size, &width);
shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE);
int32_t size = 0;
int32_t width = 0;
shellGetPrevCharSize(cmd->command, cmd->cursorOffset, &size, &width);
memmove(cmd->command + cmd->cursorOffset - size, cmd->command + cmd->cursorOffset,
cmd->commandSize - cmd->cursorOffset);
cmd->commandSize -= size;
cmd->cursorOffset -= size;
cmd->screenOffset -= width;
cmd->endOffset -= width;
showOnScreen(cmd);
shellShowOnScreen(cmd);
}
}
void clearLineBefore(Command *cmd) {
void shellClearLineBefore(SShellCmd *cmd) {
assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset);
clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size);
shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE);
memmove(cmd->command, cmd->command + cmd->cursorOffset, cmd->commandSize - cmd->cursorOffset);
cmd->commandSize -= cmd->cursorOffset;
cmd->cursorOffset = 0;
cmd->screenOffset = 0;
cmd->endOffset = cmd->commandSize;
showOnScreen(cmd);
shellShowOnScreen(cmd);
}
void clearLineAfter(Command *cmd) {
void shellClearLineAfter(SShellCmd *cmd) {
assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset);
clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size);
shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE);
cmd->commandSize -= cmd->endOffset - cmd->cursorOffset;
cmd->endOffset = cmd->cursorOffset;
showOnScreen(cmd);
shellShowOnScreen(cmd);
}
void deleteChar(Command *cmd) {
void shellDeleteChar(SShellCmd *cmd) {
assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset);
if (cmd->cursorOffset < cmd->commandSize) {
clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size);
int size = 0;
int width = 0;
getNextCharSize(cmd->command, cmd->cursorOffset, &size, &width);
shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE);
int32_t size = 0;
int32_t width = 0;
shellGetNextCharSize(cmd->command, cmd->cursorOffset, &size, &width);
memmove(cmd->command + cmd->cursorOffset, cmd->command + cmd->cursorOffset + size,
cmd->commandSize - cmd->cursorOffset - size);
cmd->commandSize -= size;
cmd->endOffset -= width;
showOnScreen(cmd);
shellShowOnScreen(cmd);
}
}
void moveCursorLeft(Command *cmd) {
void shellMoveCursorLeft(SShellCmd *cmd) {
assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset);
if (cmd->cursorOffset > 0) {
clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size);
int size = 0;
int width = 0;
getPrevCharSize(cmd->command, cmd->cursorOffset, &size, &width);
shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE);
int32_t size = 0;
int32_t width = 0;
shellGetPrevCharSize(cmd->command, cmd->cursorOffset, &size, &width);
cmd->cursorOffset -= size;
cmd->screenOffset -= width;
showOnScreen(cmd);
shellShowOnScreen(cmd);
}
}
void moveCursorRight(Command *cmd) {
void shellMoveCursorRight(SShellCmd *cmd) {
assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset);
if (cmd->cursorOffset < cmd->commandSize) {
clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size);
int size = 0;
int width = 0;
getNextCharSize(cmd->command, cmd->cursorOffset, &size, &width);
shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE);
int32_t size = 0;
int32_t width = 0;
shellGetNextCharSize(cmd->command, cmd->cursorOffset, &size, &width);
cmd->cursorOffset += size;
cmd->screenOffset += width;
showOnScreen(cmd);
shellShowOnScreen(cmd);
}
}
void positionCursorHome(Command *cmd) {
void shellPositionCursorHome(SShellCmd *cmd) {
assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset);
if (cmd->cursorOffset > 0) {
clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size);
shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE);
cmd->cursorOffset = 0;
cmd->screenOffset = 0;
showOnScreen(cmd);
shellShowOnScreen(cmd);
}
}
void positionCursorEnd(Command *cmd) {
void shellPositionCursorEnd(SShellCmd *cmd) {
assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset);
if (cmd->cursorOffset < cmd->commandSize) {
clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size);
shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE);
cmd->cursorOffset = cmd->commandSize;
cmd->screenOffset = cmd->endOffset;
showOnScreen(cmd);
shellShowOnScreen(cmd);
}
}
void printChar(char c, int times) {
for (int i = 0; i < times; i++) {
void shellPrintChar(char c, int32_t times) {
for (int32_t i = 0; i < times; i++) {
fprintf(stdout, "%c", c);
}
fflush(stdout);
}
void positionCursor(int step, int direction) {
void shellPositionCursor(int32_t step, int32_t direction) {
if (step > 0) {
if (direction == LEFT) {
fprintf(stdout, "\033[%dD", step);
@ -211,32 +311,32 @@ void positionCursor(int step, int direction) {
}
}
void updateBuffer(Command *cmd) {
void shellUpdateBuffer(SShellCmd *cmd) {
assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset);
if (regex_match(cmd->buffer, "(\\s+$)|(^$)", REG_EXTENDED)) strcat(cmd->command, " ");
if (shellRegexMatch(cmd->buffer, "(\\s+$)|(^$)", REG_EXTENDED)) strcat(cmd->command, " ");
strcat(cmd->buffer, cmd->command);
cmd->bufferSize += cmd->commandSize;
memset(cmd->command, 0, MAX_COMMAND_SIZE);
memset(cmd->command, 0, SHELL_MAX_COMMAND_SIZE);
cmd->cursorOffset = 0;
cmd->screenOffset = 0;
cmd->commandSize = 0;
cmd->endOffset = 0;
showOnScreen(cmd);
shellShowOnScreen(cmd);
}
int isReadyGo(Command *cmd) {
int32_t shellIsReadyGo(SShellCmd *cmd) {
assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset);
char *total = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE);
memset(cmd->command + cmd->commandSize, 0, MAX_COMMAND_SIZE - cmd->commandSize);
char *total = (char *)taosMemoryCalloc(1, SHELL_MAX_COMMAND_SIZE);
memset(cmd->command + cmd->commandSize, 0, SHELL_MAX_COMMAND_SIZE - cmd->commandSize);
sprintf(total, "%s%s", cmd->buffer, cmd->command);
char *reg_str =
"(^.*;\\s*$)|(^\\s*$)|(^\\s*exit\\s*$)|(^\\s*q\\s*$)|(^\\s*quit\\s*$)|(^"
"\\s*clear\\s*$)";
if (regex_match(total, reg_str, REG_EXTENDED | REG_ICASE)) {
if (shellRegexMatch(total, reg_str, REG_EXTENDED | REG_ICASE)) {
taosMemoryFree(total);
return 1;
}
@ -245,28 +345,268 @@ int isReadyGo(Command *cmd) {
return 0;
}
void getMbSizeInfo(const char *str, int *size, int *width) {
TdWchar *wc = (TdWchar *)taosMemoryCalloc(sizeof(TdWchar), MAX_COMMAND_SIZE);
void shellGetMbSizeInfo(const char *str, int32_t *size, int32_t *width) {
TdWchar *wc = (TdWchar *)taosMemoryCalloc(sizeof(TdWchar), SHELL_MAX_COMMAND_SIZE);
*size = strlen(str);
taosMbsToWchars(wc, str, MAX_COMMAND_SIZE);
*width = taosWcharsWidth(wc, MAX_COMMAND_SIZE);
taosMbsToWchars(wc, str, SHELL_MAX_COMMAND_SIZE);
*width = taosWcharsWidth(wc, SHELL_MAX_COMMAND_SIZE);
taosMemoryFree(wc);
}
void resetCommand(Command *cmd, const char s[]) {
void shellResetCommand(SShellCmd *cmd, const char s[]) {
assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset);
clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size);
memset(cmd->buffer, 0, MAX_COMMAND_SIZE);
memset(cmd->command, 0, MAX_COMMAND_SIZE);
strncpy(cmd->command, s, MAX_COMMAND_SIZE);
int size = 0;
int width = 0;
getMbSizeInfo(s, &size, &width);
shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE);
memset(cmd->buffer, 0, SHELL_MAX_COMMAND_SIZE);
memset(cmd->command, 0, SHELL_MAX_COMMAND_SIZE);
strncpy(cmd->command, s, SHELL_MAX_COMMAND_SIZE);
int32_t size = 0;
int32_t width = 0;
shellGetMbSizeInfo(s, &size, &width);
cmd->bufferSize = 0;
cmd->commandSize = size;
cmd->cursorOffset = size;
cmd->screenOffset = width;
cmd->endOffset = width;
showOnScreen(cmd);
shellShowOnScreen(cmd);
}
void shellClearScreen(int32_t ecmd_pos, int32_t cursor_pos) {
struct winsize w;
if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) {
// fprintf(stderr, "No stream device, and use default value(col 120, row 30)\n");
w.ws_col = 120;
w.ws_row = 30;
}
int32_t cursor_x = cursor_pos / w.ws_col;
int32_t cursor_y = cursor_pos % w.ws_col;
int32_t command_x = ecmd_pos / w.ws_col;
shellPositionCursor(cursor_y, LEFT);
shellPositionCursor(command_x - cursor_x, DOWN);
fprintf(stdout, "\033[2K");
for (int32_t i = 0; i < command_x; i++) {
shellPositionCursor(1, UP);
fprintf(stdout, "\033[2K");
}
fflush(stdout);
}
void shellShowOnScreen(SShellCmd *cmd) {
struct winsize w;
if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) {
fprintf(stderr, "No stream device\n");
w.ws_col = 120;
w.ws_row = 30;
}
TdWchar wc;
int32_t size = 0;
// Print out the command.
char *total_string = taosMemoryMalloc(SHELL_MAX_COMMAND_SIZE);
memset(total_string, '\0', SHELL_MAX_COMMAND_SIZE);
if (strcmp(cmd->buffer, "") == 0) {
sprintf(total_string, "%s%s", shell.info.promptHeader, cmd->command);
} else {
sprintf(total_string, "%s%s", shell.info.promptContinue, cmd->command);
}
int32_t remain_column = w.ws_col;
for (char *str = total_string; size < cmd->commandSize + PSIZE;) {
int32_t ret = taosMbToWchar(&wc, str, MB_CUR_MAX);
if (ret < 0) break;
size += ret;
/* assert(size >= 0); */
int32_t width = taosWcharWidth(wc);
if (remain_column > width) {
printf("%lc", wc);
remain_column -= width;
} else {
if (remain_column == width) {
printf("%lc\n\r", wc);
remain_column = w.ws_col;
} else {
printf("\n\r%lc", wc);
remain_column = w.ws_col - width;
}
}
str = total_string + size;
}
taosMemoryFree(total_string);
// Position the cursor
int32_t cursor_pos = cmd->screenOffset + PSIZE;
int32_t ecmd_pos = cmd->endOffset + PSIZE;
int32_t cursor_x = cursor_pos / w.ws_col;
int32_t cursor_y = cursor_pos % w.ws_col;
// int32_t cursor_y = cursor % w.ws_col;
int32_t command_x = ecmd_pos / w.ws_col;
int32_t command_y = ecmd_pos % w.ws_col;
// int32_t command_y = (command.size() + PSIZE) % w.ws_col;
shellPositionCursor(command_y, LEFT);
shellPositionCursor(command_x, UP);
shellPositionCursor(cursor_x, DOWN);
shellPositionCursor(cursor_y, RIGHT);
fflush(stdout);
}
int32_t shellReadCommand(char *command) {
SShellHistory *pHistory = &shell.history;
SShellCmd cmd = {0};
uint32_t hist_counter = pHistory->hend;
char utf8_array[10] = "\0";
cmd.buffer = (char *)taosMemoryCalloc(1, SHELL_MAX_COMMAND_SIZE);
cmd.command = (char *)taosMemoryCalloc(1, SHELL_MAX_COMMAND_SIZE);
shellShowOnScreen(&cmd);
// Read input.
char c;
while (1) {
c = (char)getchar(); // getchar() return an 'int32_t' value
if (c == EOF) {
return c;
}
if (c < 0) { // For UTF-8
int32_t count = shellCountPrefixOnes(c);
utf8_array[0] = c;
for (int32_t k = 1; k < count; k++) {
c = (char)getchar();
utf8_array[k] = c;
}
shellInsertChar(&cmd, utf8_array, count);
} else if (c < '\033') {
// Ctrl keys. TODO: Implement ctrl combinations
switch (c) {
case 1: // ctrl A
shellPositionCursorHome(&cmd);
break;
case 3:
printf("\n");
shellResetCommand(&cmd, "");
kill(0, SIGINT);
break;
case 4: // EOF or Ctrl+D
printf("\n");
return -1;
case 5: // ctrl E
shellPositionCursorEnd(&cmd);
break;
case 8:
shellBackspaceChar(&cmd);
break;
case '\n':
case '\r':
printf("\n");
if (shellIsReadyGo(&cmd)) {
sprintf(command, "%s%s", cmd.buffer, cmd.command);
taosMemoryFreeClear(cmd.buffer);
taosMemoryFreeClear(cmd.command);
return 0;
} else {
shellUpdateBuffer(&cmd);
}
break;
case 11: // Ctrl + K;
shellClearLineAfter(&cmd);
break;
case 12: // Ctrl + L;
system("clear");
shellShowOnScreen(&cmd);
break;
case 21: // Ctrl + U;
shellClearLineBefore(&cmd);
break;
}
} else if (c == '\033') {
c = (char)getchar();
switch (c) {
case '[':
c = (char)getchar();
switch (c) {
case 'A': // Up arrow
if (hist_counter != pHistory->hstart) {
hist_counter = (hist_counter + SHELL_MAX_HISTORY_SIZE - 1) % SHELL_MAX_HISTORY_SIZE;
shellResetCommand(&cmd, (pHistory->hist[hist_counter] == NULL) ? "" : pHistory->hist[hist_counter]);
}
break;
case 'B': // Down arrow
if (hist_counter != pHistory->hend) {
int32_t next_hist = (hist_counter + 1) % SHELL_MAX_HISTORY_SIZE;
if (next_hist != pHistory->hend) {
shellResetCommand(&cmd, (pHistory->hist[next_hist] == NULL) ? "" : pHistory->hist[next_hist]);
} else {
shellResetCommand(&cmd, "");
}
hist_counter = next_hist;
}
break;
case 'C': // Right arrow
shellMoveCursorRight(&cmd);
break;
case 'D': // Left arrow
shellMoveCursorLeft(&cmd);
break;
case '1':
if ((c = (char)getchar()) == '~') {
// Home key
shellPositionCursorHome(&cmd);
}
break;
case '2':
if ((c = (char)getchar()) == '~') {
// Insert key
}
break;
case '3':
if ((c = (char)getchar()) == '~') {
// Delete key
shellDeleteChar(&cmd);
}
break;
case '4':
if ((c = (char)getchar()) == '~') {
// End key
shellPositionCursorEnd(&cmd);
}
break;
case '5':
if ((c = (char)getchar()) == '~') {
// Page up key
}
break;
case '6':
if ((c = (char)getchar()) == '~') {
// Page down key
}
break;
case 72:
// Home key
shellPositionCursorHome(&cmd);
break;
case 70:
// End key
shellPositionCursorEnd(&cmd);
break;
}
break;
}
} else if (c == 0x7f) {
// press delete key
shellBackspaceChar(&cmd);
} else {
shellInsertChar(&cmd, &c, 1);
}
}
return 0;
}
#endif

731
tools/shell/src/shellEngine.c
View File

File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More