xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into feature/tq

This commit is contained in:
Liu Jicong 2022-04-25 11:42:02 +08:00
parent ecf00f7f57 7468d948f5
commit 3992effb00
83 changed files with 3323 additions and 4426 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Jenkinsfile vendored
View File

@ -113,7 +113,7 @@ pipeline {
'''
sh'''
cd ${WKC}/debug
ctest
ctest -VV
'''
}
}

2
Jenkinsfile2
View File

@ -121,7 +121,7 @@ pipeline {
pre_test()
sh'''
cd ${WKC}/debug
ctest
ctest -VV
'''
sh'''
export LD_LIBRARY_PATH=${WKC}/debug/build/lib

95
contrib/CMakeLists.txt
View File

@ -14,24 +14,9 @@ if(${BUILD_PTHREAD})
cat("${TD_SUPPORT_DIR}/pthread_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif()
# iconv
if(${BUILD_WITH_ICONV})
cat("${TD_SUPPORT_DIR}/iconv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif()
# msvc regex
if(${BUILD_MSVCREGEX})
cat("${TD_SUPPORT_DIR}/msvcregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif()
# wcwidth
if(${BUILD_WCWIDTH})
cat("${TD_SUPPORT_DIR}/wcwidth_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif()
# wingetopt
if(${BUILD_WINGETOPT})
cat("${TD_SUPPORT_DIR}/wingetopt_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
# gnu regex
if(${BUILD_GNUREGEX})
cat("${TD_SUPPORT_DIR}/gnuregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif()
# googletest
@ -114,27 +99,8 @@ if(${BUILD_TEST})
target_include_directories(
gtest
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src>
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_linux>
)
if(${TD_WINDOWS})
target_include_directories(
gtest
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_win>
)
endif(${TD_WINDOWS})
if(${TD_LINUX})
target_include_directories(
gtest
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_linux>
)
endif(${TD_LINUX})
if(${TD_DARWIN})
target_include_directories(
gtest
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_darwin>
)
endif(${TD_DARWIN})
endif(${BUILD_TEST})
# cJson
@ -216,53 +182,6 @@ if(${BUILD_WITH_NURAFT})
add_subdirectory(nuraft)
endif(${BUILD_WITH_NURAFT})
# pthread
if(${BUILD_PTHREAD})
set(CMAKE_BUILD_TYPE release)
add_definitions(-DPTW32_STATIC_LIB)
add_subdirectory(pthread)
set_target_properties(libpthreadVC3 PROPERTIES OUTPUT_NAME pthread)
add_library(pthread STATIC IMPORTED GLOBAL)
SET_PROPERTY(TARGET pthread PROPERTY IMPORTED_LOCATION ${LIBRARY_OUTPUT_PATH}/pthread.lib)
endif()
# iconv
if(${BUILD_WITH_ICONV})
add_subdirectory(iconv)
endif(${BUILD_WITH_ICONV})
# wingetopt
if(${BUILD_WINGETOPT})
add_subdirectory(wingetopt)
endif(${BUILD_WINGETOPT})
# msvcregex
if(${BUILD_MSVCREGEX})
add_library(msvcregex STATIC "")
target_sources(msvcregex
PRIVATE "msvcregex/regex.c"
)
target_include_directories(msvcregex
PRIVATE "msvcregex"
)
target_link_libraries(msvcregex
INTERFACE Shell32
)
SET_TARGET_PROPERTIES(msvcregex PROPERTIES OUTPUT_NAME msvcregex)
endif(${BUILD_MSVCREGEX})
# msvcregex
if(${BUILD_WCWIDTH})
add_library(wcwidth STATIC "")
target_sources(wcwidth
PRIVATE "wcwidth/wcwidth.c"
)
target_include_directories(wcwidth
PRIVATE "wcwidth"
)
SET_TARGET_PROPERTIES(wcwidth PROPERTIES OUTPUT_NAME wcwidth)
endif(${BUILD_WCWIDTH})
# CRAFT
if(${BUILD_WITH_CRAFT})
add_library(craft STATIC IMPORTED GLOBAL)
@ -319,12 +238,8 @@ if(${BUILD_WITH_SQLITE})
target_link_libraries(sqlite
INTERFACE m
INTERFACE pthread
INTERFACE dl
)
if(NOT TD_WINDOWS)
target_link_libraries(sqlite
INTERFACE dl
)
endif(NOT TD_WINDOWS)
endif(${BUILD_WITH_SQLITE})
# pthread

7
contrib/test/craft/CMakeLists.txt
View File

@ -1,9 +1,2 @@
add_executable(simulate_vnode "simulate_vnode.c")
target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a)
if(${BUILD_WINGETOPT})
target_link_libraries(simulate_vnode PUBLIC wingetopt)
target_include_directories(
simulate_vnode
PUBLIC "${TD_SOURCE_DIR}/contrib/wingetopt/src"
)
endif()

72
contrib/test/tdev/src/main.c
View File

@ -6,39 +6,43 @@
#define POINTER_SHIFT(ptr, s) ((void *)(((char *)ptr) + (s)))
#define POINTER_DISTANCE(pa, pb) ((char *)(pb) - (char *)(pa))
static inline void tPutA(void **buf, uint64_t val) {
memcpy(buf, &val, sizeof(val));
*buf = POINTER_SHIFT(buf, sizeof(val));
}
#define tPutA(buf, val) \
({ \
memcpy(buf, &val, sizeof(val)); \
POINTER_SHIFT(buf, sizeof(val)); \
})
static inline void tPutB(void **buf, uint64_t val) {
((uint8_t *)buf)[7] = ((val) >> 56) & 0xff;
((uint8_t *)buf)[6] = ((val) >> 48) & 0xff;
((uint8_t *)buf)[5] = ((val) >> 40) & 0xff;
((uint8_t *)buf)[4] = ((val) >> 32) & 0xff;
((uint8_t *)buf)[3] = ((val) >> 24) & 0xff;
((uint8_t *)buf)[2] = ((val) >> 16) & 0xff;
((uint8_t *)buf)[1] = ((val) >> 8) & 0xff;
((uint8_t *)buf)[0] = (val)&0xff;
*buf = POINTER_SHIFT(buf, sizeof(val));
}
#define tPutB(buf, val) \
({ \
((uint8_t *)buf)[7] = ((val) >> 56) & 0xff; \
((uint8_t *)buf)[6] = ((val) >> 48) & 0xff; \
((uint8_t *)buf)[5] = ((val) >> 40) & 0xff; \
((uint8_t *)buf)[4] = ((val) >> 32) & 0xff; \
((uint8_t *)buf)[3] = ((val) >> 24) & 0xff; \
((uint8_t *)buf)[2] = ((val) >> 16) & 0xff; \
((uint8_t *)buf)[1] = ((val) >> 8) & 0xff; \
((uint8_t *)buf)[0] = (val)&0xff; \
POINTER_SHIFT(buf, sizeof(val)); \
})
static inline void tPutC(void **buf, uint64_t val) {
if (buf) {
((uint64_t *)buf)[0] = (val);
POINTER_SHIFT(buf, sizeof(val));
}
*buf = NULL;
}
#define tPutC(buf, val) \
({ \
if (buf) { \
((uint64_t *)buf)[0] = (val); \
POINTER_SHIFT(buf, sizeof(val)); \
} \
NULL; \
})
static inline void tPutD(void **buf, uint64_t val) {
uint64_t tmp = val;
for (size_t i = 0; i < sizeof(val); i++) {
((uint8_t *)buf)[i] = tmp & 0xff;
tmp >>= 8;
}
*buf = POINTER_SHIFT(buf, sizeof(val));
}
#define tPutD(buf, val) \
({ \
uint64_t tmp = val; \
for (size_t i = 0; i < sizeof(val); i++) { \
((uint8_t *)buf)[i] = tmp & 0xff; \
tmp >>= 8; \
} \
POINTER_SHIFT(buf, sizeof(val)); \
})
static inline void tPutE(void **buf, uint64_t val) {
if (buf) {
@ -57,7 +61,7 @@ static void func(T t) {
switch (t) {
case A:
for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
tPutA(pBuf, val);
pBuf = tPutA(pBuf, val);
if (POINTER_DISTANCE(buf, pBuf) == 1024) {
pBuf = buf;
}
@ -65,7 +69,7 @@ static void func(T t) {
break;
case B:
for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
tPutB(pBuf, val);
pBuf = tPutB(pBuf, val);
if (POINTER_DISTANCE(buf, pBuf) == 1024) {
pBuf = buf;
}
@ -73,7 +77,7 @@ static void func(T t) {
break;
case C:
for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
tPutC(pBuf, val);
pBuf = tPutC(pBuf, val);
if (POINTER_DISTANCE(buf, pBuf) == 1024) {
pBuf = buf;
}
@ -81,7 +85,7 @@ static void func(T t) {
break;
case D:
for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
tPutD(pBuf, val);
pBuf = tPutD(pBuf, val);
if (POINTER_DISTANCE(buf, pBuf) == 1024) {
pBuf = buf;
}

6
examples/c/asyncdemo.c
View File

@ -45,7 +45,7 @@ typedef struct {
void taos_insert_call_back(void *param, TAOS_RES *tres, int code);
void taos_select_call_back(void *param, TAOS_RES *tres, int code);
void taos_error(TAOS *taos);
void shellPrintError(TAOS *taos);
static void queryDB(TAOS *taos, char *command) {
int i;
@ -102,7 +102,7 @@ int main(int argc, char *argv[])
taos = taos_connect(argv[1], "root", "taosdata", NULL, 0);
if (taos == NULL)
taos_error(taos);
shellPrintError(taos);
printf("success to connect to server\n");
@ -193,7 +193,7 @@ int main(int argc, char *argv[])
return 0;
}
void taos_error(TAOS *con)
void shellPrintError(TAOS *con)
{
fprintf(stderr, "TDengine error: %s\n", taos_errstr(con));
taos_close(con);

13
include/common/tdatablock.h
View File

@ -238,10 +238,16 @@ static FORCE_INLINE int32_t blockCompressColData(SColumnInfoData* pColRes, int32
static FORCE_INLINE void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols,
int8_t needCompress) {
int32_t* colSizes = (int32_t*)data;
int32_t* actualLen = (int32_t*) data;
data += sizeof(int32_t);
uint64_t* groupId = (uint64_t*) data;
data += sizeof(uint64_t);
int32_t* colSizes = (int32_t*)data;
data += numOfCols * sizeof(int32_t);
*dataLen = (numOfCols * sizeof(int32_t));
*dataLen = (numOfCols * sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t));
int32_t numOfRows = pBlock->info.rows;
for (int32_t col = 0; col < numOfCols; ++col) {
@ -273,6 +279,9 @@ static FORCE_INLINE void blockCompressEncode(const SSDataBlock* pBlock, char* da
colSizes[col] = htonl(colSizes[col]);
}
*actualLen = *dataLen;
*groupId = pBlock->info.groupId;
}
#ifdef __cplusplus

7
include/common/tglobal.h
View File

@ -34,13 +34,9 @@ extern int32_t tsVersion;
extern int32_t tsStatusInterval;
// common
extern int32_t tsRpcTimer;
extern int32_t tsRpcMaxTime;
extern bool tsRpcForceTcp; // all commands go to tcp protocol if this is enabled
extern int32_t tsMaxConnections;
extern int32_t tsMaxShellConns;
extern int32_t tsShellActivityTimer;
extern int32_t tsMaxTmrCtrl;
extern int32_t tsCompressMsgSize;
extern int32_t tsCompressColData;
extern int32_t tsMaxNumOfDistinctResults;
@ -98,9 +94,6 @@ extern bool tsDeadLockKillQuery;
extern int32_t tsQueryPolicy;
// client
extern int32_t tsMaxWildCardsLen;
extern int32_t tsMaxRegexStringLen;
extern int32_t tsMaxNumOfOrderedResults;
extern int32_t tsMinSlidingTime;
extern int32_t tsMinIntervalTime;
extern int32_t tsMaxStreamComputDelay;

1
include/common/tmsgdef.h
View File

@ -85,6 +85,7 @@ enum {
TD_DEF_MSG_TYPE(TDMT_DND_DROP_VNODE, "dnode-drop-vnode", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_DND_CONFIG_DNODE, "dnode-config-dnode", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_DND_SERVER_STATUS, "dnode-server-status", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_DND_NET_TEST, "dnode-net-test", NULL, NULL)
// Requests handled by MNODE
TD_NEW_MSG_SEG(TDMT_MND_MSG)

2
include/libs/catalog/catalog.h
View File

@ -51,7 +51,7 @@ typedef struct SMetaData {
SArray *pTableMeta; // STableMeta array
SArray *pVgroupInfo; // SVgroupInfo list
SArray *pUdfList; // udf info list
SArray *pEpSetList; // qnode epset list, SArray<SEpSet>
SArray *pQnodeList; // qnode list, SArray<SQueryNodeAddr>
} SMetaData;
typedef struct SCatalogCfg {

35
include/libs/function/function.h
View File

@ -37,7 +37,7 @@ typedef struct SFuncExecEnv {
typedef bool (*FExecGetEnv)(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
typedef bool (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo);
typedef int32_t (*FExecProcess)(struct SqlFunctionCtx *pCtx);
typedef int32_t (*FExecFinalize)(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock, int32_t slotId);
typedef int32_t (*FExecFinalize)(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock);
typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
typedef struct SScalarFuncExecFuncs {
@ -141,8 +141,7 @@ struct SResultRowEntryInfo;
//for selectivity query, the corresponding tag value is assigned if the data is qualified
typedef struct SSubsidiaryResInfo {
int16_t bufLen; // keep the tags data for top/bottom query result
int16_t numOfCols;
int16_t num;
struct SqlFunctionCtx **pCtx;
} SSubsidiaryResInfo;
@ -187,8 +186,8 @@ typedef struct SqlFunctionCtx {
uint8_t currentStage; // record current running step, default: 0
bool isAggSet;
int64_t startTs; // timestamp range of current query when function is executed on a specific data block, TODO remove it
/////////////////////////////////////////////////////////////////
bool stableQuery;
/////////////////////////////////////////////////////////////////
int16_t functionId; // function id
char * pOutput; // final result output buffer, point to sdata->data
int32_t numOfParams;
@ -198,11 +197,15 @@ typedef struct SqlFunctionCtx {
int32_t offset;
SVariant tag;
struct SResultRowEntryInfo *resultInfo;
SSubsidiaryResInfo subsidiaryRes;
SPoint1 start;
SPoint1 end;
SFuncExecFuncs fpSet;
SScalarFuncExecFuncs sfp;
SSubsidiaryResInfo subsidiaries;
SPoint1 start;
SPoint1 end;
SFuncExecFuncs fpSet;
SScalarFuncExecFuncs sfp;
SExprInfo *pExpr;
struct SDiskbasedBuf *pBuf;
struct SSDataBlock *pSrcBlock;
int32_t curBufPage;
} SqlFunctionCtx;
enum {
@ -319,6 +322,20 @@ struct SUdfInfo;
void qAddUdfInfo(uint64_t id, struct SUdfInfo* pUdfInfo);
void qRemoveUdfInfo(uint64_t id, struct SUdfInfo* pUdfInfo);
/**
* create udfd proxy, called once in process that call setupUdf/callUdfxxx/teardownUdf
* @return error code
*/
int32_t udfcOpen();
/**
* destroy udfd proxy
* @return error code
*/
int32_t udfcClose();
typedef void *UdfcFuncHandle;
#ifdef __cplusplus
}
#endif

14
include/os/osSystem.h
View File

@ -31,19 +31,19 @@ extern "C" {
typedef struct TdCmd *TdCmdPtr;
TdCmdPtr taosOpenCmd(const char *cmd);
int64_t taosGetLineCmd(TdCmdPtr pCmd, char ** __restrict ptrBuf);
int32_t taosEOFCmd(TdCmdPtr pCmd);
int64_t taosCloseCmd(TdCmdPtr *ppCmd);
TdCmdPtr taosOpenCmd(const char* cmd);
int64_t taosGetLineCmd(TdCmdPtr pCmd, char** __restrict ptrBuf);
int32_t taosEOFCmd(TdCmdPtr pCmd);
int64_t taosCloseCmd(TdCmdPtr* ppCmd);
void* taosLoadDll(const char* filename);
void* taosLoadSym(void* handle, char* name);
void taosCloseDll(void* handle);
int32_t taosSetConsoleEcho(bool on);
void setTerminalMode();
int32_t getOldTerminalMode();
void resetTerminalMode();
void taosSetTerminalMode();
int32_t taosGetOldTerminalMode();
void taosResetTerminalMode();
#ifdef __cplusplus
}

2
include/util/tdef.h
View File

@ -76,8 +76,6 @@ extern const int32_t TYPE_BYTES[15];
#define TSDB_DEFAULT_PASS "taosdata"
#endif
#define SHELL_MAX_PASSWORD_LEN 20
#define TSDB_TRUE 1
#define TSDB_FALSE 0
#define TSDB_OK 0

19
source/client/src/clientImpl.c
View File

@ -105,9 +105,9 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass,
epSet.epSet.eps[0].port = port;
}
char* key = getClusterKey(user, secretEncrypt, ip, port);
SAppInstInfo** pInst = NULL;
char* key = getClusterKey(user, secretEncrypt, ip, port);
SAppInstInfo** pInst = NULL;
taosThreadMutexLock(&appInfo.mutex);
pInst = taosHashGet(appInfo.pInstMap, key, strlen(key));
@ -840,10 +840,21 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32
return code;
}
int32_t* colLength = (int32_t*)pResultInfo->pData;
char* pStart = ((char*)pResultInfo->pData) + sizeof(int32_t) * numOfCols;
char* p = (char*) pResultInfo->pData;
int32_t dataLen = *(int32_t*) p;
p += sizeof(int32_t);
uint64_t groupId = *(uint64_t*) p;
p += sizeof(uint64_t);
int32_t* colLength = (int32_t*)p;
p += sizeof(int32_t) * numOfCols;
char* pStart = p;
for (int32_t i = 0; i < numOfCols; ++i) {
colLength[i] = htonl(colLength[i]);
ASSERT(colLength[i] < dataLen);
if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) {
pResultInfo->pCol[i].offset = (int32_t*)pStart;

5
source/common/src/tdatablock.c
View File

@ -644,12 +644,12 @@ size_t blockDataGetRowSize(SSDataBlock* pBlock) {
/**
* @refitem blockDataToBuf for the meta size
*
* @param pBlock
* @return
*/
size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock) {
return sizeof(int32_t) + pBlock->info.numOfCols * sizeof(int32_t);
// | total rows/total length | block group id | each column length |
return sizeof(int32_t) + sizeof(uint64_t) + pBlock->info.numOfCols * sizeof(int32_t);
}
double blockDataGetSerialRowSize(const SSDataBlock* pBlock) {
@ -1219,7 +1219,6 @@ void colDataDestroy(SColumnInfoData* pColData) {
taosMemoryFree(pColData->pData);
}
static void doShiftBitmap(char* nullBitmap, size_t n, size_t total) {
int32_t len = BitmapLen(total);

31
source/common/src/tglobal.c
View File

@ -32,13 +32,9 @@ int32_t tsVersion = 30000000;
int32_t tsStatusInterval = 1; // second
// common
int32_t tsRpcTimer = 300;
int32_t tsRpcMaxTime = 600; // seconds;
bool tsRpcForceTcp = true; // disable this, means query, show command use udp protocol as default
int32_t tsMaxShellConns = 50000;
int32_t tsMaxConnections = 50000;
int32_t tsShellActivityTimer = 3; // second
int32_t tsMaxBinaryDisplayWidth = 30;
bool tsEnableSlaveQuery = true;
bool tsPrintAuth = false;
@ -105,14 +101,6 @@ int32_t tsCompressColData = -1;
*/
int32_t tsCompatibleModel = 1;
// client
int32_t tsMaxWildCardsLen = TSDB_PATTERN_STRING_DEFAULT_LEN;
int32_t tsMaxRegexStringLen = TSDB_REGEX_STRING_DEFAULT_LEN;
// the maximum number of results for projection query on super table that are returned from
// one virtual node, to order according to timestamp
int32_t tsMaxNumOfOrderedResults = 100000;
// 10 ms for sliding time, the value will changed in case of time precision changed
int32_t tsMinSlidingTime = 10;
@ -314,19 +302,10 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, 1) != 0) return -1;
if (cfgAddDir(pCfg, "tempDir", tsTempDir, 1) != 0) return -1;
if (cfgAddFloat(pCfg, "minimalTempDirGB", 1.0f, 0.001f, 10000000, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "maxTmrCtrl", tsMaxTmrCtrl, 8, 2048, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "rpcTimer", tsRpcTimer, 100, 3000, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "rpcMaxTime", tsRpcMaxTime, 100, 7200, 1) != 0) return -1;
if (cfgAddBool(pCfg, "rpcForceTcp", tsRpcForceTcp, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "shellActivityTimer", tsShellActivityTimer, 1, 120, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "compressMsgSize", tsCompressMsgSize, -1, 100000000, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "compressColData", tsCompressColData, -1, 100000000, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "maxWildCardsLength", tsMaxWildCardsLen, 0, TSDB_MAX_FIELD_LEN, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "maxRegexStringLen", tsMaxRegexStringLen, 0, TSDB_MAX_FIELD_LEN, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "maxNumOfOrderedRes", tsMaxNumOfOrderedResults, 128, TSDB_MAX_ALLOWED_SQL_LEN, 1) != 0)
return -1;
if (cfgAddBool(pCfg, "keepColumnName", tsKeepOriginalColumnName, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "maxBinaryDisplayWidth", tsMaxBinaryDisplayWidth, 1, 65536, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 3, 1) != 0) return -1;
tsNumOfTaskQueueThreads = tsNumOfCores / 4;
@ -508,18 +487,10 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
return -1;
}
tsMaxTmrCtrl = cfgGetItem(pCfg, "maxTmrCtrl")->i32;
tsRpcTimer = cfgGetItem(pCfg, "rpcTimer")->i32;
tsRpcMaxTime = cfgGetItem(pCfg, "rpcMaxTime")->i32;
tsRpcForceTcp = cfgGetItem(pCfg, "rpcForceTcp")->i32;
tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32;
tsCompressMsgSize = cfgGetItem(pCfg, "compressMsgSize")->i32;
tsCompressColData = cfgGetItem(pCfg, "compressColData")->i32;
tsMaxWildCardsLen = cfgGetItem(pCfg, "maxWildCardsLength")->i32;
tsMaxRegexStringLen = cfgGetItem(pCfg, "maxRegexStringLen")->i32;
tsMaxNumOfOrderedResults = cfgGetItem(pCfg, "maxNumOfOrderedRes")->i32;
tsKeepOriginalColumnName = cfgGetItem(pCfg, "keepColumnName")->bval;
tsMaxBinaryDisplayWidth = cfgGetItem(pCfg, "maxBinaryDisplayWidth")->i32;
tsNumOfTaskQueueThreads = cfgGetItem(pCfg, "numOfTaskQueueThreads")->i32;
tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32;
return 0;
@ -710,6 +681,6 @@ void taosCfgDynamicOptions(const char *option, const char *value) {
if (strcasecmp(option, "resetlog") == 0) {
taosResetLog();
cfgDumpCfg(tsCfg, 1, false);
cfgDumpCfg(tsCfg, 0, false);
}
}

26
source/common/src/tmsg.c
View File

@ -1567,13 +1567,8 @@ int32_t tSerializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pReq
if (tEncodeI32(&encoder, pReq->codeLen) < 0) return -1;
if (tEncodeI64(&encoder, pReq->signature) < 0) return -1;
int32_t codeSize = 0;
if (pReq->pCode != NULL) {
codeSize = strlen(pReq->pCode) + 1;
}
if (tEncodeI32(&encoder, codeSize) < 0) return -1;
if (pReq->pCode != NULL) {
if (tEncodeCStr(&encoder, pReq->pCode) < 0) return -1;
if (tEncodeBinary(&encoder, pReq->pCode, pReq->codeLen) < 0) return -1;
}
int32_t commentSize = 0;
@ -1607,10 +1602,8 @@ int32_t tDeserializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pR
if (tDecodeI32(&decoder, &pReq->codeLen) < 0) return -1;
if (tDecodeI64(&decoder, &pReq->signature) < 0) return -1;
int32_t codeSize = 0;
if (tDecodeI32(&decoder, &codeSize) < 0) return -1;
if (codeSize > 0) {
pReq->pCode = taosMemoryCalloc(1, codeSize);
if (pReq->codeLen > 0) {
pReq->pCode = taosMemoryCalloc(1, pReq->codeLen);
if (pReq->pCode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
@ -1733,7 +1726,7 @@ int32_t tSerializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp *
if (tEncodeI32(&encoder, pInfo->codeSize) < 0) return -1;
if (tEncodeI32(&encoder, pInfo->commentSize) < 0) return -1;
if (pInfo->codeSize) {
if (tEncodeCStr(&encoder, pInfo->pCode) < 0) return -1;
if (tEncodeBinary(&encoder, pInfo->pCode, pInfo->codeSize) < 0) return -1;
}
if (pInfo->commentSize) {
if (tEncodeCStr(&encoder, pInfo->pComment) < 0) return -1;
@ -2090,10 +2083,15 @@ int32_t tDeserializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp
if (tStartDecode(&decoder) < 0) return -1;
int32_t num = 0;
if (tDecodeI32(&decoder, &num) < 0) return -1;
pRsp->addrsList = taosArrayInit(num, sizeof(SQueryNodeAddr));
if (NULL == pRsp->addrsList) return -1;
if (NULL == pRsp->addrsList) {
pRsp->addrsList = taosArrayInit(num, sizeof(SQueryNodeAddr));
if (NULL == pRsp->addrsList) return -1;
}
for (int32_t i = 0; i < num; ++i) {
if (tDecodeSQueryNodeAddr(&decoder, TARRAY_GET_ELEM(pRsp->addrsList, i)) < 0) return -1;
SQueryNodeAddr addr = {0};
if (tDecodeSQueryNodeAddr(&decoder, &addr) < 0) return -1;
taosArrayPush(pRsp->addrsList, &addr);
}
tEndDecode(&decoder);

2
source/dnode/mgmt/exe/dmMain.c
View File

@ -106,7 +106,7 @@ static void dmPrintVersion() {
static void dmDumpCfg() {
SConfig *pCfg = taosGetCfg();
cfgDumpCfg(pCfg, 0, 1);
cfgDumpCfg(pCfg, 0, true);
}
static SDnodeOpt dmGetOpt() {

24
source/dnode/mgmt/implement/src/dmHandle.c
View File

@ -311,6 +311,9 @@ static void dmWatchUdfd(void *args) {
}
static int32_t dmStartUdfd(SDnode *pDnode) {
char dnodeId[8] = {0};
snprintf(dnodeId, sizeof(dnodeId), "%d", pDnode->data.dnodeId);
uv_os_setenv("DNODE_ID", dnodeId);
SUdfdData *pData = &pDnode->udfdData;
if (pData->startCalled) {
dInfo("dnode-mgmt start udfd already called");
@ -320,8 +323,17 @@ static int32_t dmStartUdfd(SDnode *pDnode) {
uv_barrier_init(&pData->barrier, 2);
uv_thread_create(&pData->thread, dmWatchUdfd, pDnode);
uv_barrier_wait(&pData->barrier);
pData->needCleanUp = true;
return pData->spawnErr;
int32_t err = atomic_load_32(&pData->spawnErr);
if (err != 0) {
uv_barrier_destroy(&pData->barrier);
uv_async_send(&pData->stopAsync);
uv_thread_join(&pData->thread);
pData->needCleanUp = false;
dInfo("dnode-mgmt udfd cleaned up after spawn err");
} else {
pData->needCleanUp = true;
}
return err;
}
static int32_t dmStopUdfd(SDnode *pDnode) {
@ -336,7 +348,7 @@ static int32_t dmStopUdfd(SDnode *pDnode) {
uv_barrier_destroy(&pData->barrier);
uv_async_send(&pData->stopAsync);
uv_thread_join(&pData->thread);
dInfo("dnode-mgmt udfd cleaned up");
return 0;
}
@ -371,9 +383,9 @@ static int32_t dmInitMgmt(SMgmtWrapper *pWrapper) {
}
dmReportStartup(pDnode, "dnode-transport", "initialized");
// if (dmStartUdfd(pDnode) != 0) {
// dError("failed to start udfd");
// }
if (dmStartUdfd(pDnode) != 0) {
dError("failed to start udfd");
}
dInfo("dnode-mgmt is initialized");
return 0;

6
source/dnode/mgmt/implement/src/dmTransport.c
View File

@ -140,6 +140,12 @@ static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
return;
}
if (msgType == TDMT_DND_NET_TEST) {
dTrace("net test req will be processed, handle:%p, app:%p", pMsg->handle, pMsg->ahandle);
dmProcessServerStatusReq(pDnode, pMsg);
return;
}
if (pDnode->status != DND_STAT_RUNNING) {
dError("msg:%s ignored since dnode not running, handle:%p app:%p", TMSG_INFO(msgType), pMsg->handle, pMsg->ahandle);
if (isReq) {

1
source/dnode/mgmt/interface/inc/dmInt.h
View File

@ -37,6 +37,7 @@ void dmSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgF
void dmReportStartup(SDnode *pDnode, const char *pName, const char *pDesc);
void dmReportStartupByWrapper(SMgmtWrapper *pWrapper, const char *pName, const char *pDesc);
void dmProcessServerStatusReq(SDnode *pDnode, SRpcMsg *pMsg);
void dmProcessNettestReq(SDnode *pDnode, SRpcMsg *pMsg);
void dmGetMonitorSysInfo(SMonSysInfo *pInfo);
// dmFile.c

12
source/dnode/mgmt/interface/src/dmInt.c
View File

@ -171,6 +171,18 @@ static void dmGetServerStatus(SDnode *pDnode, SServerStatusRsp *pStatus) {
}
}
void dmProcessNettestReq(SDnode *pDnode, SRpcMsg *pRpc) {
dDebug("net test req is received");
SRpcMsg rsp = {.handle = pRpc->handle, .refId = pRpc->refId, .ahandle = pRpc->ahandle, .code = 0};
rsp.pCont = rpcMallocCont(pRpc->contLen);
if (rsp.pCont == NULL) {
rsp.code = TSDB_CODE_OUT_OF_MEMORY;
} else {
rsp.contLen = pRpc->contLen;
}
rpcSendResponse(&rsp);
}
void dmProcessServerStatusReq(SDnode *pDnode, SRpcMsg *pReq) {
dDebug("server status req is received");

1
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -177,6 +177,7 @@ void mmInitMsgHandle(SMgmtWrapper *pWrapper) {
dmSetMsgHandle(pWrapper, TDMT_MND_DROP_MNODE, mmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_QNODE, mmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_MND_DROP_QNODE, mmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_MND_QNODE_LIST, mmProcessReadMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_SNODE, mmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_MND_DROP_SNODE, mmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_BNODE, mmProcessWriteMsg, DEFAULT_HANDLE);

7
source/dnode/mgmt/mgmt_vnode/src/vmInt.c
View File

@ -15,6 +15,7 @@
#define _DEFAULT_SOURCE
#include "vmInt.h"
#include "libs/function/function.h"
SVnodeObj *vmAcquireVnode(SVnodesMgmt *pMgmt, int32_t vgId) {
SVnodeObj *pVnode = NULL;
@ -275,7 +276,7 @@ static void vmCleanup(SMgmtWrapper *pWrapper) {
pWrapper->pMgmt = NULL;
// syncCleanUp();
udfcClose();
dInfo("vnode-mgmt is cleaned up");
}
@ -339,6 +340,10 @@ static int32_t vmInit(SMgmtWrapper *pWrapper) {
}
dmReportStartup(pDnode, "vnode-vnodes", "initialized");
if (udfcOpen() != 0) {
dError("failed to open udfc in dnode");
}
code = 0;
_OVER:

8
source/dnode/mnode/impl/src/mndFunc.c
View File

@ -309,10 +309,10 @@ static int32_t mndProcessCreateFuncReq(SNodeMsg *pReq) {
goto _OVER;
}
if (createReq.pCode[0] == 0) {
terrno = TSDB_CODE_MND_INVALID_FUNC_CODE;
goto _OVER;
}
if (createReq.codeLen <= 1) {
terrno = TSDB_CODE_MND_INVALID_FUNC_CODE;
goto _OVER;
}
if (createReq.bufSize <= 0 || createReq.bufSize > TSDB_FUNC_BUF_SIZE) {
terrno = TSDB_CODE_MND_INVALID_FUNC_BUFSIZE;

5
source/dnode/mnode/impl/src/mndQnode.c
View File

@ -451,8 +451,9 @@ static int32_t mndProcessQnodeListReq(SNodeMsg *pReq) {
goto _OVER;
}
void *pIter = NULL;
while (1) {
void *pIter = sdbFetch(pSdb, SDB_QNODE, NULL, (void **)&pObj);
pIter = sdbFetch(pSdb, SDB_QNODE, pIter, (void **)&pObj);
if (pIter == NULL) break;
SQueryNodeAddr nodeAddr = {0};
@ -472,7 +473,7 @@ static int32_t mndProcessQnodeListReq(SNodeMsg *pReq) {
}
int32_t rspLen = tSerializeSQnodeListRsp(NULL, 0, &qlistRsp);
void *pRsp = taosMemoryMalloc(rspLen);
void *pRsp = rpcMallocCont(rspLen);
if (pRsp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;

6
source/dnode/mnode/impl/src/mnode.c
View File

@ -385,7 +385,11 @@ int32_t mndProcessMsg(SNodeMsg *pMsg) {
terrno = code;
mTrace("msg:%p, in progress, app:%p", pMsg, ahandle);
} else if (code != 0) {
mError("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle);
if (terrno != TSDB_CODE_OPS_NOT_SUPPORT) {
mError("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle);
} else {
mTrace("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle);
}
} else {
mTrace("msg:%p, is processed, app:%p", pMsg, ahandle);
}

88
source/dnode/mnode/impl/test/func/func.cpp
View File

@ -22,17 +22,16 @@ class MndTestFunc : public ::testing::Test {
void SetUp() override {}
void TearDown() override {}
void SetCode(SCreateFuncReq* pReq, const char* pCode);
void SetCode(SCreateFuncReq* pReq, const char* pCode, int32_t size);
void SetComment(SCreateFuncReq* pReq, const char* pComment);
};
Testbase MndTestFunc::test;
void MndTestFunc::SetCode(SCreateFuncReq* pReq, const char* pCode) {
int32_t len = strlen(pCode);
pReq->pCode = (char*)taosMemoryCalloc(1, len + 1);
strcpy(pReq->pCode, pCode);
pReq->codeLen = len;
void MndTestFunc::SetCode(SCreateFuncReq *pReq, const char *pCode, int32_t size) {
pReq->pCode = (char*)taosMemoryMalloc(size);
memcpy(pReq->pCode, pCode, size);
pReq->codeLen = size;
}
void MndTestFunc::SetComment(SCreateFuncReq* pReq, const char* pComment) {
@ -79,7 +78,7 @@ TEST_F(MndTestFunc, 02_Create_Func) {
{
SCreateFuncReq createReq = {0};
strcpy(createReq.name, "f1");
SetCode(&createReq, "");
SetCode(&createReq, "", 1);
SetComment(&createReq, "comment1");
int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq);
@ -95,7 +94,7 @@ TEST_F(MndTestFunc, 02_Create_Func) {
{
SCreateFuncReq createReq = {0};
strcpy(createReq.name, "f1");
SetCode(&createReq, "code1");
SetCode(&createReq, "code1", 6);
SetComment(&createReq, "comment1");
int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq);
@ -111,7 +110,7 @@ TEST_F(MndTestFunc, 02_Create_Func) {
{
SCreateFuncReq createReq = {0};
strcpy(createReq.name, "f1");
SetCode(&createReq, "code1");
SetCode(&createReq, "code1", 6);
SetComment(&createReq, "comment1");
createReq.bufSize = TSDB_FUNC_BUF_SIZE + 1;
@ -128,7 +127,7 @@ TEST_F(MndTestFunc, 02_Create_Func) {
for (int32_t i = 0; i < 3; ++i) {
SCreateFuncReq createReq = {0};
strcpy(createReq.name, "f1");
SetCode(&createReq, "code1");
SetCode(&createReq, "code1", 6);
SetComment(&createReq, "comment1");
createReq.bufSize = TSDB_FUNC_BUF_SIZE + 1;
createReq.igExists = 0;
@ -253,7 +252,7 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) {
createReq.outputLen = 24;
createReq.bufSize = 6;
createReq.signature = 18;
SetCode(&createReq, "code2");
SetCode(&createReq, "code2", 6);
SetComment(&createReq, "comment2");
int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq);
@ -439,3 +438,70 @@ TEST_F(MndTestFunc, 04_Drop_Func) {
test.SendShowReq(TSDB_MGMT_TABLE_FUNC, "user_functions", "");
EXPECT_EQ(test.GetShowRows(), 1);
}
TEST_F(MndTestFunc, 05_Actual_code) {
{
SCreateFuncReq createReq = {0};
strcpy(createReq.name, "udf1");
char code[300] = {0};
for (int32_t i = 0; i < sizeof(code); ++i) {
code[i] = (i) % 20;
}
SetCode(&createReq, code, 300);
SetComment(&createReq, "comment1");
createReq.bufSize = 8;
createReq.igExists = 0;
createReq.funcType = 1;
createReq.scriptType = 2;
createReq.outputType = TSDB_DATA_TYPE_SMALLINT;
createReq.outputLen = 12;
createReq.bufSize = 4;
createReq.signature = 5;
int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq);
void* pReq = rpcMallocCont(contLen);
tSerializeSCreateFuncReq(pReq, contLen, &createReq);
tFreeSCreateFuncReq(&createReq);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0);
}
{
SRetrieveFuncReq retrieveReq = {0};
retrieveReq.numOfFuncs = 1;
retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN);
taosArrayPush(retrieveReq.pFuncNames, "udf1");
int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq);
void* pReq = rpcMallocCont(contLen);
tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq);
tFreeSRetrieveFuncReq(&retrieveReq);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0);
SRetrieveFuncRsp retrieveRsp = {0};
tDeserializeSRetrieveFuncRsp(pRsp->pCont, pRsp->contLen, &retrieveRsp);
EXPECT_EQ(retrieveRsp.numOfFuncs, 1);
EXPECT_EQ(retrieveRsp.numOfFuncs, (int32_t)taosArrayGetSize(retrieveRsp.pFuncInfos));
SFuncInfo* pFuncInfo = (SFuncInfo*)taosArrayGet(retrieveRsp.pFuncInfos, 0);
EXPECT_STREQ(pFuncInfo->name, "udf1");
EXPECT_EQ(pFuncInfo->funcType, 1);
EXPECT_EQ(pFuncInfo->scriptType, 2);
EXPECT_EQ(pFuncInfo->outputType, TSDB_DATA_TYPE_SMALLINT);
EXPECT_EQ(pFuncInfo->outputLen, 12);
EXPECT_EQ(pFuncInfo->bufSize, 4);
EXPECT_EQ(pFuncInfo->signature, 5);
EXPECT_STREQ("comment1", pFuncInfo->pComment);
for (int32_t i = 0; i < 300; ++i) {
EXPECT_EQ(pFuncInfo->pCode[i], (i) % 20);
}
tFreeSRetrieveFuncRsp(&retrieveRsp);
}
}

7
source/dnode/qnode/src/qnode.c
View File

@ -17,6 +17,7 @@
#include "qndInt.h"
#include "query.h"
#include "qworker.h"
//#include "tudf.h"
SQnode *qndOpen(const SQnodeOpt *pOption) {
SQnode *pQnode = taosMemoryCalloc(1, sizeof(SQnode));
@ -25,6 +26,8 @@ SQnode *qndOpen(const SQnodeOpt *pOption) {
return NULL;
}
//udfcOpen();
if (qWorkerInit(NODE_TYPE_QNODE, pQnode->qndId, NULL, (void **)&pQnode->pQuery, &pOption->msgCb)) {
taosMemoryFreeClear(pQnode);
return NULL;
@ -37,13 +40,15 @@ SQnode *qndOpen(const SQnodeOpt *pOption) {
void qndClose(SQnode *pQnode) {
qWorkerDestroy((void **)&pQnode->pQuery);
//udfcClose();
taosMemoryFree(pQnode);
}
int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad) { return 0; }
int32_t qndProcessQueryMsg(SQnode *pQnode, SRpcMsg *pMsg) {
qTrace("message in query queue is processing");
qTrace("message in qnode query queue is processing");
SReadHandle handle = {0};
switch (pMsg->msgType) {

9
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -3223,8 +3223,13 @@ void tsdbRetrieveDataBlockInfo(tsdbReaderT* pTsdbReadHandle, SDataBlockInfo* pDa
tsdbDebug("data block generated, uid:%" PRIu64 " numOfRows:%d, tsrange:%" PRId64 " - %" PRId64 " %s", uid, cur->rows,
cur->win.skey, cur->win.ekey, pHandle->idStr);
// pDataBlockInfo->uid = uid; // block Id may be over write by assigning uid fro this data block. Do NOT assign
// the table uid
pDataBlockInfo->uid = uid;
#if 0
// for multi-group data query processing test purpose
pDataBlockInfo->groupId = uid;
#endif
pDataBlockInfo->rows = cur->rows;
pDataBlockInfo->window = cur->win;
pDataBlockInfo->numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pHandle));

1
source/dnode/vnode/src/tsdb/tsdbSma.c
View File

@ -1510,7 +1510,6 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, char *pData, int64_t indexUid,
}
STSma *pTSma = pItem->pSma;
#endif
STSmaReadH tReadH = {0};

2
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -135,7 +135,7 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg
}
int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) {
vTrace("message in query queue is processing");
vTrace("message in vnode query queue is processing");
SReadHandle handle = {.reader = pVnode->pTsdb, .meta = pVnode->pMeta, .config = &pVnode->config};
switch (pMsg->msgType) {

9
source/libs/catalog/src/catalog.c
View File

@ -494,7 +494,7 @@ _return:
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, SArray **out) {
int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, SArray *out) {
char *msg = NULL;
int32_t msgLen = 0;
@ -526,7 +526,7 @@ int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmt
CTG_ERR_RET(code);
}
ctgDebug("Got qnode list from mnode, listNum:%d", (int32_t)taosArrayGetSize(*out));
ctgDebug("Got qnode list from mnode, listNum:%d", (int32_t)taosArrayGetSize(out));
return TSDB_CODE_SUCCESS;
}
@ -2778,7 +2778,8 @@ int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps,
}
if (pReq->qNodeRequired) {
CTG_ERR_JRET(ctgGetQnodeListFromMnode(pCtg, pTrans, pMgmtEps, &pRsp->pEpSetList));
pRsp->pQnodeList = taosArrayInit(10, sizeof(SQueryNodeAddr));
CTG_ERR_JRET(ctgGetQnodeListFromMnode(pCtg, pTrans, pMgmtEps, pRsp->pQnodeList));
}
CTG_API_LEAVE(TSDB_CODE_SUCCESS);
@ -2807,7 +2808,7 @@ int32_t catalogGetQnodeList(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps,
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
}
CTG_ERR_JRET(ctgGetQnodeListFromMnode(pCtg, pRpc, pMgmtEps, &pQnodeList));
CTG_ERR_JRET(ctgGetQnodeListFromMnode(pCtg, pRpc, pMgmtEps, pQnodeList));
_return:

40
source/libs/command/src/explain.c
View File

@ -304,8 +304,8 @@ int32_t qExplainResAppendRow(SExplainCtx *ctx, char *tbuf, int32_t len, int32_t
memcpy(row.buf, tbuf, len);
row.level = level;
row.len = len;
ctx->dataSize += len;
row.len = len;
ctx->dataSize += row.len;
if (NULL == taosArrayPush(ctx->rows, &row)) {
qError("taosArrayPush row to explain res rows failed");
@ -756,7 +756,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
}
int32_t colNum = 1;
int32_t rspSize = sizeof(SRetrieveTableRsp) + sizeof(int32_t) * colNum + sizeof(int32_t) * rowNum + pCtx->dataSize;
int32_t rspSize = sizeof(SRetrieveTableRsp) + sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t) * colNum + sizeof(int32_t) * rowNum + pCtx->dataSize;
SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize);
if (NULL == rsp) {
qError("malloc SRetrieveTableRsp failed, size:%d", rspSize);
@ -766,29 +766,38 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
rsp->completed = 1;
rsp->numOfRows = htonl(rowNum);
*(int32_t *)rsp->data = htonl(pCtx->dataSize);
// payload length
*(int32_t *)rsp->data = sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t) * colNum + sizeof(int32_t) * rowNum + pCtx->dataSize;
int32_t *offset = (int32_t *)((char *)rsp->data + sizeof(int32_t));
// group id
*(uint64_t*)(rsp->data + sizeof(int32_t)) = 0;
// column length
int32_t* colLength = (int32_t *)(rsp->data + sizeof(int32_t) + sizeof(uint64_t));
// varchar column offset segment
int32_t *offset = (int32_t *)((char *)colLength + sizeof(int32_t));
// varchar data real payload
char *data = (char *)(offset + rowNum);
int32_t tOffset = 0;
char* start = data;
for (int32_t i = 0; i < rowNum; ++i) {
SQueryExplainRowInfo *row = taosArrayGet(pCtx->rows, i);
*offset = tOffset;
tOffset += row->len;
offset[i] = data - start;
memcpy(data, row->buf, row->len);
++offset;
varDataCopy(data, row->buf);
ASSERT(varDataTLen(row->buf) == row->len);
data += row->len;
}
*pRsp = rsp;
*colLength = htonl(data - start);
rsp->compLen = htonl(rspSize);
*pRsp = rsp;
return TSDB_CODE_SUCCESS;
}
int32_t qExplainPrepareCtx(SQueryPlan *pDag, SExplainCtx **pCtx) {
int32_t code = 0;
SNodeListNode *plans = NULL;
@ -895,9 +904,7 @@ int32_t qExplainAppendPlanRows(SExplainCtx *pCtx) {
int32_t qExplainGenerateRsp(SExplainCtx *pCtx, SRetrieveTableRsp **pRsp) {
QRY_ERR_RET(qExplainAppendGroupResRows(pCtx, pCtx->rootGroupId, 0));
QRY_ERR_RET(qExplainAppendPlanRows(pCtx));
QRY_ERR_RET(qExplainGetRspFromCtx(pCtx, pRsp));
return TSDB_CODE_SUCCESS;
@ -967,13 +974,10 @@ int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp) {
SExplainCtx *pCtx = NULL;
QRY_ERR_RET(qExplainPrepareCtx(pDag, &pCtx));
QRY_ERR_JRET(qExplainGenerateRsp(pCtx, pRsp));
_return:
qExplainFreeCtx(pCtx);
QRY_RET(code);
}

11
source/libs/executor/inc/executil.h
View File

@ -40,8 +40,6 @@
#define GET_TASKID(_t) (((SExecTaskInfo*)(_t))->id.str)
#define curTimeWindowIndex(_winres) ((_winres)->curIndex)
typedef struct SGroupResInfo {
int32_t totalGroup;
int32_t currentGroup;
@ -68,11 +66,16 @@ typedef struct SResultRowPosition {
int32_t offset;
} SResultRowPosition;
typedef struct SResKeyPos {
SResultRowPosition pos;
uint64_t groupId;
char key[];
} SResKeyPos;
typedef struct SResultRowInfo {
SResultRowPosition *pPosition;
int32_t size; // number of result set
int32_t capacity; // max capacity
// int32_t curPos; // current active result row index of pResult list
SResultRowPosition cur;
} SResultRowInfo;
@ -135,7 +138,7 @@ typedef struct {
int32_t colId;
} SStddevInterResult;
void initGroupResInfo(SGroupResInfo* pGroupResInfo, SResultRowInfo* pResultInfo);
void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, bool sortGroupResult);
void initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList);
void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo);

21
source/libs/executor/inc/executorimpl.h
View File

@ -347,10 +347,11 @@ typedef struct STableScanInfo {
} STableScanInfo;
typedef struct STagScanInfo {
SColumnInfo* pCols;
SSDataBlock* pRes;
SColumnInfo *pCols;
SSDataBlock *pRes;
int32_t totalTables;
int32_t curPos;
void *pReader;
} STagScanInfo;
typedef struct SStreamBlockScanInfo {
@ -376,13 +377,11 @@ typedef struct SSysTableScanInfo {
SEpSet epSet;
tsem_t ready;
int32_t accountId;
bool showRewrite;
SNode* pCondition; // db_name filter condition, to discard data that are not in current database
void* pCur; // cursor for iterate the local table meta store.
SArray* scanCols; // SArray<int16_t> scan column id list
// int32_t type; // show type, TODO remove it
int32_t accountId;
bool showRewrite;
SNode* pCondition; // db_name filter condition, to discard data that are not in current database
void* pCur; // cursor for iterate the local table meta store.
SArray* scanCols; // SArray<int16_t> scan column id list
SName name;
SSDataBlock* pRes;
int32_t capacity;
@ -628,7 +627,7 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput,
SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfCols, int32_t dataLoadFlag, int32_t repeatTime,
SOperatorInfo* createTableScanOperatorInfo(void* pReaderHandle, int32_t order, int32_t numOfCols, int32_t dataLoadFlag, int32_t repeatTime,
int32_t reverseTime, SArray* pColMatchInfo, SSDataBlock* pResBlock, SNode* pCondition,
SInterval* pInterval, double ratio, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExprInfo* pScalarExprInfo,
@ -668,12 +667,12 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo*
SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SNode* pOnCondition, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTagScanOperatorInfo(void* pReaderHandle, SExprInfo* pExpr, int32_t numOfOutput, SExecTaskInfo* pTaskInfo);
#if 0
SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv);
SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream,
SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createTagScanOperatorInfo(SReaderHandle* pReaderHandle, SExprInfo* pExpr, int32_t numOfOutput);
#endif
void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx,

8
source/libs/executor/src/dataDispatcher.c
View File

@ -64,10 +64,10 @@ static bool needCompress(const SSDataBlock* pData, int32_t numOfCols) {
}
// data format:
// +----------------+--------------------------------------+-------------+-----------+-------------+-----------+
// |SDataCacheEntry | column#1 length, column#2 length ... | col1 bitmap | col1 data | col2 bitmap | col2 data | ....
// | | sizeof(int32_t) * numOfCols | actual size | | actual size | |
// +----------------+--------------------------------------+-------------+-----------+-------------+-----------+
// +----------------+--------------+----------+--------------------------------------+-------------+-----------+-------------+-----------+
// |SDataCacheEntry | total length | group id | column#1 length, column#2 length ... | col1 bitmap | col1 data | col2 bitmap | col2 data | ....
// | | (4 bytes) |(8 bytes) | sizeof(int32_t) * numOfCols | actual size | | actual size | |
// +----------------+--------------+----------+--------------------------------------+-------------+-----------+-------------+-----------+
// The length of bitmap is decided by number of rows of this data block, and the length of each column data is
// recorded in the first segment, next to the struct header
static void toDataCacheEntry(const SDataDispatchHandle* pHandle, const SInputData* pInput, SDataDispatchBuf* pBuf) {

42
source/libs/executor/src/executil.c
View File

@ -186,12 +186,50 @@ void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo) {
pGroupResInfo->index = 0;
}
void initGroupResInfo(SGroupResInfo* pGroupResInfo, SResultRowInfo* pResultInfo) {
static int32_t resultrowCompar1(const void* p1, const void* p2) {
SResKeyPos* pp1 = *(SResKeyPos**) p1;
SResKeyPos* pp2 = *(SResKeyPos**) p2;
if (pp1->groupId == pp2->groupId) {
int64_t pts1 = *(int64_t*) pp1->key;
int64_t pts2 = *(int64_t*) pp2->key;
if (pts1 == pts2) {
return 0;
} else {
return pts1 < pts2? -1:1;
}
} else {
return pp1->groupId < pp2->groupId? -1:1;
}
}
void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, bool sortGroupResult) {
if (pGroupResInfo->pRows != NULL) {
taosArrayDestroy(pGroupResInfo->pRows);
}
pGroupResInfo->pRows = taosArrayFromList(pResultInfo->pPosition, pResultInfo->size, sizeof(SResultRowPosition));
// extract the result rows information from the hash map
void* pData = NULL;
pGroupResInfo->pRows = taosArrayInit(10, POINTER_BYTES);
size_t keyLen = 0;
while((pData = taosHashIterate(pHashmap, pData)) != NULL) {
void* key = taosHashGetKey(pData, &keyLen);
SResKeyPos* p = taosMemoryMalloc(keyLen + sizeof(SResultRowPosition));
p->groupId = *(uint64_t*) key;
p->pos = *(SResultRowPosition*) pData;
memcpy(p->key, key + sizeof(uint64_t), keyLen - sizeof(uint64_t));
taosArrayPush(pGroupResInfo->pRows, &p);
}
if (sortGroupResult) {
qsort(pGroupResInfo->pRows->pData, taosArrayGetSize(pGroupResInfo->pRows), POINTER_BYTES, resultrowCompar1);
}
pGroupResInfo->index = 0;
assert(pGroupResInfo->index <= getNumOfTotalRes(pGroupResInfo));
}

394
source/libs/executor/src/executorimpl.c
View File

@ -235,7 +235,6 @@ static int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDi
static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size);
static void setResultBufSize(STaskAttr* pQueryAttr, SResultInfo* pResultInfo);
static void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable);
static void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, uint64_t groupId,
SExecTaskInfo* pTaskInfo);
@ -298,26 +297,6 @@ SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode) {
return pBlock;
}
static bool isSelectivityWithTagsQuery(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
return true;
// bool hasTags = false;
// int32_t numOfSelectivity = 0;
//
// for (int32_t i = 0; i < numOfOutput; ++i) {
// int32_t functId = pCtx[i].functionId;
// if (functId == FUNCTION_TAG_DUMMY || functId == FUNCTION_TS_DUMMY) {
// hasTags = true;
// continue;
// }
//
// if ((aAggs[functId].status & FUNCSTATE_SELECTIVITY) != 0) {
// numOfSelectivity++;
// }
// }
//
// return (numOfSelectivity > 0 && hasTags);
}
static bool hasNull(SColumn* pColumn, SColumnDataAgg* pStatis) {
if (TSDB_COL_IS_TAG(pColumn->flag) || TSDB_COL_IS_UD_COL(pColumn->flag) ||
pColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
@ -433,6 +412,13 @@ SResultRow* getNewResultRow_rv(SDiskbasedBuf* pResultBuf, int64_t tableGroupId,
return pResultRow;
}
/**
* the struct of key in hash table
* +----------+---------------+
* | group id | key data |
* | 8 bytes | actual length |
* +----------+---------------+
*/
static SResultRow* doSetResultOutBufByKey_rv(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, int64_t uid,
char* pData, int16_t bytes, bool masterscan, uint64_t groupId,
SExecTaskInfo* pTaskInfo, bool isIntervalQuery, SAggSupporter* pSup) {
@ -1098,8 +1084,9 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt
int32_t code = TSDB_CODE_SUCCESS;
for (int32_t i = 0; i < pOperator->numOfOutput; ++i) {
pCtx[i].order = order;
pCtx[i].size = pBlock->info.rows;
pCtx[i].order = order;
pCtx[i].size = pBlock->info.rows;
pCtx[i].pSrcBlock = pBlock;
pCtx[i].currentStage = MAIN_SCAN;
SInputColumnInfoData* pInput = &pCtx[i].input;
@ -1474,7 +1461,7 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe
SArray* pUpdated = NULL;
if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) {
pUpdated = taosArrayInit(4, sizeof(SResultRowPosition));
pUpdated = taosArrayInit(4, POINTER_BYTES);
}
int32_t step = 1;
@ -1486,8 +1473,6 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe
if (pSDataBlock->pDataBlock != NULL) {
SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
tsCols = (int64_t*)pColDataInfo->pData;
// assert(tsCols[0] == pSDataBlock->info.window.skey && tsCols[pSDataBlock->info.rows - 1] ==
// pSDataBlock->info.window.ekey);
}
int32_t startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1);
@ -1506,7 +1491,11 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe
}
if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) {
SResultRowPosition pos = {.pageId = pResult->pageId, .offset = pResult->offset};
SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
pos->groupId = tableGroupId;
pos->pos = (SResultRowPosition) {.pageId = pResult->pageId, .offset = pResult->offset};
*(int64_t*) pos->key = pResult->win.skey;
taosArrayPush(pUpdated, &pos);
}
@ -1580,7 +1569,11 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe
}
if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) {
SResultRowPosition pos = {.pageId = pResult->pageId, .offset = pResult->offset};
SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
pos->groupId = tableGroupId;
pos->pos = (SResultRowPosition) {.pageId = pResult->pageId, .offset = pResult->offset};
*(int64_t*) pos->key = pResult->win.skey;
taosArrayPush(pUpdated, &pos);
}
@ -1844,10 +1837,6 @@ void setBlockStatisInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock*
// set the output buffer for the selectivity + tag query
static int32_t setCtxTagColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
if (!isSelectivityWithTagsQuery(pCtx, numOfOutput)) {
return TSDB_CODE_SUCCESS;
}
int32_t num = 0;
int16_t tagLen = 0;
@ -1874,9 +1863,8 @@ static int32_t setCtxTagColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
}
}
if (p != NULL) {
p->subsidiaryRes.pCtx = pTagCtx;
p->subsidiaryRes.numOfCols = num;
p->subsidiaryRes.bufLen = tagLen;
p->subsidiaries.pCtx = pTagCtx;
p->subsidiaries.num = num;
} else {
taosMemoryFreeClear(pTagCtx);
}
@ -1903,6 +1891,9 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput,
SqlFunctionCtx* pCtx = &pFuncCtx[i];
pCtx->functionId = -1;
pCtx->curBufPage = -1;
pCtx->pExpr = pExpr;
if (pExpr->pExpr->nodeType == QUERY_NODE_FUNCTION) {
SFuncExecEnv env = {0};
pCtx->functionId = pExpr->pExpr->_function.pFunctNode->funcId;
@ -1930,9 +1921,9 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput,
pCtx->pTsOutput = NULL;
pCtx->resDataInfo.bytes = pFunct->resSchema.bytes;
pCtx->resDataInfo.type = pFunct->resSchema.type;
pCtx->order = TSDB_ORDER_ASC;
pCtx->order = TSDB_ORDER_ASC;
pCtx->start.key = INT64_MIN;
pCtx->end.key = INT64_MIN;
pCtx->end.key = INT64_MIN;
pCtx->numOfParams = pExpr->base.numOfParams;
pCtx->param = pFunct->pParam;
@ -1992,7 +1983,7 @@ static void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
}
taosVariantDestroy(&pCtx[i].tag);
taosMemoryFreeClear(pCtx[i].subsidiaryRes.pCtx);
taosMemoryFreeClear(pCtx[i].subsidiaries.pCtx);
}
taosMemoryFreeClear(pCtx);
@ -2095,25 +2086,6 @@ static int32_t updateBlockLoadStatus(STaskAttr* pQuery, int32_t status) {
return status;
}
static void doUpdateLastKey(STaskAttr* pQueryAttr) {
STimeWindow* win = &pQueryAttr->window;
size_t num = taosArrayGetSize(pQueryAttr->tableGroupInfo.pGroupList);
for (int32_t i = 0; i < num; ++i) {
SArray* p1 = taosArrayGetP(pQueryAttr->tableGroupInfo.pGroupList, i);
size_t len = taosArrayGetSize(p1);
for (int32_t j = 0; j < len; ++j) {
// STableKeyInfo* pInfo = taosArrayGet(p1, j);
//
// // update the new lastkey if it is equalled to the value of the old skey
// if (pInfo->lastKey == win->ekey) {
// pInfo->lastKey = win->skey;
// }
}
}
}
// static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableReq* pQueryMsg, bool stableQuery) {
// STaskAttr* pQueryAttr = pQInfo->runtimeEnv.pQueryAttr;
//
@ -2848,6 +2820,7 @@ void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) {
}
}
// todo merged with the build group result.
void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDiskbasedBuf* pBuf,
SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) {
for (int32_t i = 0; i < pResultRowInfo->size; ++i) {
@ -2882,40 +2855,36 @@ void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SD
}
}
// todo merged with the build group result.
void finalizeUpdatedResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDiskbasedBuf* pBuf, SArray* pUpdateList,
int32_t* rowCellInfoOffset) {
size_t num = taosArrayGetSize(pUpdateList);
for (int32_t i = 0; i < num; ++i) {
SResultRowPosition* pPos = taosArrayGet(pUpdateList, i);
SFilePage* bufPage = getBufPage(pBuf, pPos->pageId);
SResultRow* pRow = (SResultRow*)((char*)bufPage + pPos->offset);
SResKeyPos * pPos = taosArrayGetP(pUpdateList, i);
SFilePage* bufPage = getBufPage(pBuf, pPos->pos.pageId);
SResultRow* pRow = (SResultRow*)((char*)bufPage + pPos->pos.offset);
//
for (int32_t j = 0; j < numOfOutput; ++j) {
pCtx[j].resultInfo = getResultCell(pRow, j, rowCellInfoOffset);
//
struct SResultRowEntryInfo* pResInfo = pCtx[j].resultInfo;
if (isRowEntryCompleted(pResInfo) && isRowEntryInitialized(pResInfo)) {
continue;
}
if (pCtx[j].fpSet.process) { // TODO set the dummy function.
// pCtx[j].fpSet.finalize(&pCtx[j]);
pResInfo->initialized = true;
}
// if (isRowEntryCompleted(pResInfo) && isRowEntryInitialized(pResInfo)) {
// continue;
// }
//
// if (pCtx[j].fpSet.process) { // TODO set the dummy function.
//// pCtx[j].fpSet.finalize(&pCtx[j]);
// pResInfo->initialized = true;
// }
//
if (pRow->numOfRows < pResInfo->numOfRes) {
pRow->numOfRows = pResInfo->numOfRes;
}
}
releaseBufPage(pBuf, bufPage);
/*
* set the number of output results for group by normal columns, the number of output rows usually is 1 except
* the top and bottom query
*/
// buf->numOfRows = (uint16_t)getNumOfResult(pCtx, numOfOutput);
}
}
@ -3062,33 +3031,6 @@ void setExecutionContext(int32_t numOfOutput, uint64_t groupId, SExecTaskInfo* p
pAggInfo->groupId = groupId;
}
void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable) {
STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
SExprBasicInfo* pExpr = &pExprInfo->base;
// if (pQueryAttr->stableQuery && (pRuntimeEnv->pTsBuf != NULL) &&
// (pExpr->functionId == FUNCTION_TS || pExpr->functionId == FUNCTION_PRJ) &&
// (pExpr->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_ID)) {
// assert(pExpr->numOfParams == 1);
//
// int16_t tagColId = (int16_t)pExprInfo->base.param[0].i;
// SColumnInfo* pColInfo = doGetTagColumnInfoById(pQueryAttr->tagColList, pQueryAttr->numOfTags, tagColId);
//
// doSetTagValueInParam(pTable, tagColId, &pCtx->tag, pColInfo->type, pColInfo->bytes);
//
// int16_t tagType = pCtx[0].tag.nType;
// if (tagType == TSDB_DATA_TYPE_BINARY || tagType == TSDB_DATA_TYPE_NCHAR) {
// //qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%s",
// GET_TASKID(pRuntimeEnv),
//// pExprInfo->base.param[0].i, pCtx[0].tag.pz);
// } else {
// //qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%" PRId64,
// GET_TASKID(pRuntimeEnv),
//// pExprInfo->base.param[0].i, pCtx[0].tag.i);
// }
// }
}
/*
* There are two cases to handle:
*
@ -3131,7 +3073,6 @@ void setIntervalQueryRange(STableQueryInfo* pTableQueryInfo, TSKEY key, STimeWin
}
/**
* copyToOutputBuf support copy data in ascending/descending order
* For interval query of both super table and table, copy the data in ascending order, since the output results are
* ordered in SWindowResutl already. While handling the group by query for both table and super table,
* all group result are completed already.
@ -3159,10 +3100,10 @@ int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbased
}
for (int32_t i = start; (i < numOfRows) && (i >= 0); i += step) {
SResultRowPosition* pPos = taosArrayGet(pGroupResInfo->pRows, i);
SFilePage* page = getBufPage(pBuf, pPos->pageId);
SResKeyPos *pPos = taosArrayGetP(pGroupResInfo->pRows, i);
SFilePage* page = getBufPage(pBuf, pPos->pos.pageId);
SResultRow* pRow = (SResultRow*)((char*)page + pPos->offset);
SResultRow* pRow = (SResultRow*)((char*)page + pPos->pos.offset);
if (pRow->numOfRows == 0) {
pGroupResInfo->index += 1;
continue;
@ -3181,7 +3122,7 @@ int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbased
pCtx[j].resultInfo = getResultCell(pRow, j, rowCellOffset);
if (pCtx[j].fpSet.process) {
pCtx[j].fpSet.finalize(&pCtx[j], pBlock, slotId);
pCtx[j].fpSet.finalize(&pCtx[j], pBlock);
} else {
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId);
@ -3806,9 +3747,15 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI
blockDataEnsureCapacity(pRes, numOfRows);
if (pColList == NULL) { // data from other sources
int32_t* colLen = (int32_t*)pData;
char* pStart = pData + sizeof(int32_t) * numOfOutput;
int32_t dataLen = *(int32_t*) pData;
pData += sizeof(int32_t);
pRes->info.groupId = *(uint64_t*) pData;
pData += sizeof(uint64_t);
int32_t* colLen = (int32_t*)pData;
char* pStart = pData + sizeof(int32_t) * numOfOutput;
for (int32_t i = 0; i < numOfOutput; ++i) {
colLen[i] = htonl(colLen[i]);
ASSERT(colLen[i] > 0);
@ -3861,7 +3808,11 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI
blockDataEnsureCapacity(&block, numOfRows);
int32_t* colLen = (int32_t*)pStart;
int32_t dataLen = *(int32_t*) pStart;
uint64_t groupId = *(uint64_t*) (pStart + sizeof(int32_t));
pStart += sizeof(int32_t) + sizeof(uint64_t);
int32_t* colLen = (int32_t*) (pStart);
pStart += sizeof(int32_t) * numOfCols;
for (int32_t i = 0; i < numOfCols; ++i) {
@ -3903,6 +3854,7 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI
}
pRes->info.rows = numOfRows;
blockDataUpdateTsWindow(pRes);
int64_t el = taosGetTimestampUs() - startTs;
@ -4318,18 +4270,6 @@ static void assignExprInfo(SExprInfo* dst, const SExprInfo* src) {
// }
}
static SExprInfo* exprArrayDup(SArray* pExprList) {
size_t numOfOutput = taosArrayGetSize(pExprList);
SExprInfo* p = taosMemoryCalloc(numOfOutput, sizeof(SExprInfo));
for (int32_t i = 0; i < numOfOutput; ++i) {
SExprInfo* pExpr = taosArrayGetP(pExprList, i);
assignExprInfo(&p[i], pExpr);
}
return p;
}
// TODO merge aggregate super table
static void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) {
for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
@ -4822,7 +4762,7 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) {
finalizeMultiTupleQueryResult(pAggInfo->binfo.pCtx, pOperator->numOfOutput, pAggInfo->aggSup.pResultBuf,
&pAggInfo->binfo.resultRowInfo, pAggInfo->binfo.rowCellInfoOffset);
initGroupResInfo(&pAggInfo->groupResInfo, &pAggInfo->binfo.resultRowInfo);
initGroupedResultInfo(&pAggInfo->groupResInfo, pAggInfo->aggSup.pResultRowHashTable, false);
OPTR_SET_OPENED(pOperator);
return TSDB_CODE_SUCCESS;
}
@ -5125,7 +5065,7 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) {
STableQueryInfo* pTableQueryInfo = pInfo->pCurrent;
setIntervalQueryRange(pTableQueryInfo, pBlock->info.window.skey, &pTaskInfo->window);
hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, 0);
hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, pBlock->info.groupId);
#if 0 // test for encode/decode result info
if(pOperator->encodeResultRow){
@ -5147,7 +5087,7 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) {
finalizeMultiTupleQueryResult(pInfo->binfo.pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf,
&pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset);
initGroupResInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo);
initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true);
OPTR_SET_OPENED(pOperator);
return TSDB_CODE_SUCCESS;
}
@ -5276,7 +5216,7 @@ static SSDataBlock* doAllIntervalAgg(SOperatorInfo* pOperator, bool* newgroup) {
setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
// finalizeQueryResult(pSliceInfo->binfo.pCtx, pOperator->numOfOutput);
initGroupResInfo(&pSliceInfo->groupResInfo, &pSliceInfo->binfo.resultRowInfo);
// initGroupedResultInfo(&pSliceInfo->groupResInfo, &pSliceInfo->binfo.resultRowInfo);
// doBuildResultDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pSliceInfo->pRes);
if (pSliceInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pSliceInfo->groupResInfo)) {
@ -5327,7 +5267,7 @@ static SSDataBlock* doSTableIntervalAgg(SOperatorInfo* pOperator, bool* newgroup
finalizeMultiTupleQueryResult(pInfo->binfo.pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf,
&pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset);
initGroupResInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo);
// initGroupedResultInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo);
OPTR_SET_OPENED(pOperator);
blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
@ -5458,7 +5398,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator, bool* newgroup) {
finalizeMultiTupleQueryResult(pBInfo->pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo,
pBInfo->rowCellInfoOffset);
initGroupResInfo(&pInfo->groupResInfo, &pBInfo->resultRowInfo);
initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true);
blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf,
pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx);
@ -5510,7 +5450,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator, bool* newgroup)
finalizeMultiTupleQueryResult(pBInfo->pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo,
pBInfo->rowCellInfoOffset);
initGroupResInfo(&pInfo->groupResInfo, &pBInfo->resultRowInfo);
initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true);
blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf,
pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx);
@ -5699,6 +5639,11 @@ int32_t initAggInfo(SOptrBasicInfo* pBasicInfo, SAggSupporter* pAggSup, SExprInf
pBasicInfo->pRes = pResultBlock;
doInitAggInfoSup(pAggSup, pBasicInfo->pCtx, numOfCols, keyBufSize, pkey);
for(int32_t i = 0; i < numOfCols; ++i) {
pBasicInfo->pCtx[i].pBuf = pAggSup->pResultBuf;
}
return TSDB_CODE_SUCCESS;
}
@ -5847,11 +5792,6 @@ static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) {
doDestroyBasicInfo(&pInfo->binfo, numOfOutput);
}
static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) {
STagScanInfo* pInfo = (STagScanInfo*)param;
pInfo->pRes = blockDataDestroy(pInfo->pRes);
}
static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) {
SSortOperatorInfo* pInfo = (SSortOperatorInfo*)param;
pInfo->pDataBlock = blockDataDestroy(pInfo->pDataBlock);
@ -5941,11 +5881,12 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo*
goto _error;
}
pInfo->order = TSDB_ORDER_ASC;
pInfo->interval = *pInterval;
pInfo->execModel = pTaskInfo->execModel;
pInfo->win = pTaskInfo->window;
pInfo->twAggSup = *pTwAggSupp;
pInfo->order = TSDB_ORDER_ASC;
pInfo->interval = *pInterval;
// pInfo->execModel = OPTR_EXEC_MODEL_STREAM;
pInfo->execModel = pTaskInfo->execModel;
pInfo->win = pTaskInfo->window;
pInfo->twAggSup = *pTwAggSupp;
pInfo->primaryTsIndex = primaryTsSlotId;
int32_t numOfRows = 4096;
@ -6204,157 +6145,6 @@ _error:
return NULL;
}
static SSDataBlock* doTagScan(SOperatorInfo* pOperator, bool* newgroup) {
#if 0
SOperatorInfo* pOperator = (SOperatorInfo*) param;
if (pOperator->status == OP_EXEC_DONE) {
return NULL;
}
int32_t maxNumOfTables = (int32_t)pResultInfo->capacity;
STagScanInfo *pInfo = pOperator->info;
SSDataBlock *pRes = pInfo->pRes;
*newgroup = false;
int32_t count = 0;
SArray* pa = GET_TABLEGROUP(pRuntimeEnv, 0);
int32_t functionId = getExprFunctionId(&pOperator->pExpr[0]);
if (functionId == FUNCTION_TID_TAG) { // return the tags & table Id
assert(pQueryAttr->numOfOutput == 1);
SExprInfo* pExprInfo = &pOperator->pExpr[0];
int32_t rsize = pExprInfo->base.resSchema.bytes;
count = 0;
int16_t bytes = pExprInfo->base.resSchema.bytes;
int16_t type = pExprInfo->base.resSchema.type;
for(int32_t i = 0; i < pQueryAttr->numOfTags; ++i) {
if (pQueryAttr->tagColList[i].colId == pExprInfo->base.pColumns->info.colId) {
bytes = pQueryAttr->tagColList[i].bytes;
type = pQueryAttr->tagColList[i].type;
break;
}
}
SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0);
while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) {
int32_t i = pInfo->curPos++;
STableQueryInfo *item = taosArrayGetP(pa, i);
char *output = pColInfo->pData + count * rsize;
varDataSetLen(output, rsize - VARSTR_HEADER_SIZE);
output = varDataVal(output);
STableId* id = TSDB_TABLEID(item->pTable);
*(int16_t *)output = 0;
output += sizeof(int16_t);
*(int64_t *)output = id->uid; // memory align problem, todo serialize
output += sizeof(id->uid);
*(int32_t *)output = id->tid;
output += sizeof(id->tid);
*(int32_t *)output = pQueryAttr->vgId;
output += sizeof(pQueryAttr->vgId);
char* data = NULL;
if (pExprInfo->base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX) {
data = tsdbGetTableName(item->pTable);
} else {
data = tsdbGetTableTagVal(item->pTable, pExprInfo->base.pColumns->info.colId, type, bytes);
}
doSetTagValueToResultBuf(output, data, type, bytes);
count += 1;
}
//qDebug("QInfo:0x%"PRIx64" create (tableId, tag) info completed, rows:%d", GET_TASKID(pRuntimeEnv), count);
} else if (functionId == FUNCTION_COUNT) {// handle the "count(tbname)" query
SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0);
*(int64_t*)pColInfo->pData = pInfo->totalTables;
count = 1;
pOperator->status = OP_EXEC_DONE;
//qDebug("QInfo:0x%"PRIx64" create count(tbname) query, res:%d rows:1", GET_TASKID(pRuntimeEnv), count);
} else { // return only the tags|table name etc.
SExprInfo* pExprInfo = &pOperator->pExpr[0]; // todo use the column list instead of exprinfo
count = 0;
while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) {
int32_t i = pInfo->curPos++;
STableQueryInfo* item = taosArrayGetP(pa, i);
char *data = NULL, *dst = NULL;
int16_t type = 0, bytes = 0;
for(int32_t j = 0; j < pOperator->numOfOutput; ++j) {
// not assign value in case of user defined constant output column
if (TSDB_COL_IS_UD_COL(pExprInfo[j].base.pColumns->flag)) {
continue;
}
SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, j);
type = pExprInfo[j].base.resSchema.type;
bytes = pExprInfo[j].base.resSchema.bytes;
if (pExprInfo[j].base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX) {
data = tsdbGetTableName(item->pTable);
} else {
data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.pColumns->info.colId, type, bytes);
}
dst = pColInfo->pData + count * pExprInfo[j].base.resSchema.bytes;
doSetTagValueToResultBuf(dst, data, type, bytes);
}
count += 1;
}
if (pInfo->curPos >= pInfo->totalTables) {
pOperator->status = OP_EXEC_DONE;
}
//qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_TASKID(pRuntimeEnv), count);
}
if (pOperator->status == OP_EXEC_DONE) {
setTaskStatus(pOperator->pRuntimeEnv, TASK_COMPLETED);
}
pRes->info.rows = count;
return (pRes->info.rows == 0)? NULL:pInfo->pRes;
#endif
return TSDB_CODE_SUCCESS;
}
SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput) {
STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo));
size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pRuntimeEnv);
assert(numOfGroup == 0 || numOfGroup == 1);
pInfo->curPos = 0;
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
pOperator->name = "SeqTableTagScan";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN;
pOperator->blockingOptr = false;
pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo;
pOperator->getNextFn = doTagScan;
pOperator->pExpr = pExpr;
pOperator->numOfOutput = numOfOutput;
pOperator->closeFn = destroyTagScanOperatorInfo;
return pOperator;
}
static int32_t getColumnIndexInSource(SQueriedTableInfo* pTableInfo, SExprBasicInfo* pExpr, SColumnInfo* pTagCols) {
int32_t j = 0;
@ -6562,6 +6352,9 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
int32_t numOfCols = 0;
tsdbReaderT pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableGroupInfo, (uint64_t)queryId, taskId);
if (pDataReader == NULL) {
return NULL;
}
SArray* pColList =
extractColMatchInfo(pScanPhyNode->pScanCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfCols);
@ -6576,8 +6369,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
};
return createTableScanOperatorInfo(pDataReader, pTableScanNode->scanSeq[0] > 0 ? TSDB_ORDER_ASC : TSDB_ORDER_DESC,
numOfCols, pTableScanNode->dataRequired, pTableScanNode->scanSeq[0], pTableScanNode->scanSeq[1], pColList,
pResBlock, pScanPhyNode->node.pConditions, &interval, pTableScanNode->ratio, pTaskInfo);
numOfCols, pTableScanNode->dataRequired, pTableScanNode->scanSeq[0], pTableScanNode->scanSeq[1], pColList,
pResBlock, pScanPhyNode->node.pConditions, &interval, pTableScanNode->ratio, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == type) {
SExchangePhysiNode* pExchange = (SExchangePhysiNode*)pPhyNode;
SSDataBlock* pResBlock = createResDataBlock(pExchange->node.pOutputDataBlockDesc);
@ -6585,17 +6378,14 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
} else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) {
SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table.
int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo,
queryId, taskId);
int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, queryId, taskId);
SArray* tableIdList = extractTableIdList(pTableGroupInfo);
SSDataBlock* pResBlock = createResDataBlock(pScanPhyNode->node.pOutputDataBlockDesc);
int32_t numOfCols = 0;
SArray* pColList =
extractColMatchInfo(pScanPhyNode->pScanCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfCols);
SOperatorInfo* pOperator =
createStreamScanOperatorInfo(pHandle->reader, pResBlock, pColList, tableIdList, pTaskInfo);
SArray* pCols = extractColMatchInfo(pScanPhyNode->pScanCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfCols);
SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle->reader, pResBlock, pCols, tableIdList, pTaskInfo);
taosArrayDestroy(tableIdList);
return pOperator;
} else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) {
@ -6621,6 +6411,9 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
for (int32_t i = 0; i < size; ++i) {
SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, i);
ops[i] = createOperatorTree(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableGroupInfo);
if (ops[i] == NULL) {
return NULL;
}
}
SOperatorInfo* pOptr = NULL;
@ -6988,8 +6781,7 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead
goto _complete;
}
STableGroupInfo group = {0};
(*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &group);
(*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &(*pTaskInfo)->tableqinfoGroupInfo);
if (NULL == (*pTaskInfo)->pRoot) {
code = terrno;
goto _complete;

2
source/libs/executor/src/groupoperator.c
View File

@ -308,7 +308,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgrou
// }
blockDataEnsureCapacity(pRes, pOperator->resultInfo.capacity);
initGroupResInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo);
initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, false);
while(1) {
doBuildResultDatablock(pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx);

198
source/libs/executor/src/scanoperator.c
View File

@ -271,9 +271,7 @@ static void setupEnvForReverseScan(STableScanInfo* pTableScanInfo, SqlFunctionCt
static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator, bool* newgroup) {
STableScanInfo* pTableScanInfo = pOperator->info;
SSDataBlock* pBlock = pTableScanInfo->pResBlock;
STableGroupInfo* pTableGroupInfo = &pOperator->pTaskInfo->tableqinfoGroupInfo;
SSDataBlock* pBlock = pTableScanInfo->pResBlock;
*newgroup = false;
while (tsdbNextDataBlock(pTableScanInfo->dataReader)) {
@ -284,18 +282,6 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator, bool* newgroup) {
pTableScanInfo->numOfBlocks += 1;
tsdbRetrieveDataBlockInfo(pTableScanInfo->dataReader, &pBlock->info);
// todo opt
// if (pTableGroupInfo->numOfTables > 1 || (pRuntimeEnv->current == NULL && pTableGroupInfo->numOfTables == 1)) {
// STableQueryInfo** pTableQueryInfo =
// (STableQueryInfo**)taosHashGet(pTableGroupInfo->map, &pBlock->info.uid, sizeof(pBlock->info.uid));
// if (pTableQueryInfo == NULL) {
// break;
// }
//
// doTableQueryInfoTimeWindowCheck(pTaskInfo, *pTableQueryInfo, pTableScanInfo->order);
// }
// this function never returns error?
uint32_t status = 0;
int32_t code = loadDataBlock(pOperator, pTableScanInfo, pBlock, &status);
// int32_t code = loadDataBlockOnDemand(pOperator->pRuntimeEnv, pTableScanInfo, pBlock, &status);
@ -308,6 +294,8 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator, bool* newgroup) {
continue;
}
// reset the block to be 0 by default, this blockId is assigned by physical plan and is used by direct upstream operator.
pBlock->info.blockId = 0;
return pBlock;
}
@ -405,11 +393,11 @@ SOperatorInfo* createTableScanOperatorInfo(void* pDataReader, int32_t order, int
pOperator->name = "TableScanOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN;
pOperator->blockingOptr = false;
pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo;
pOperator->numOfOutput = numOfOutput;
pOperator->getNextFn = doTableScan;
pOperator->pTaskInfo = pTaskInfo;
pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo;
pOperator->numOfOutput = numOfOutput;
pOperator->getNextFn = doTableScan;
pOperator->pTaskInfo = pTaskInfo;
static int32_t cost = 0;
pOperator->cost.openCost = ++cost;
@ -824,12 +812,12 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) {
int32_t tableNameSlotId = 1;
SColumnInfoData* pTableNameCol = taosArrayGet(pInfo->pRes->pDataBlock, tableNameSlotId);
char* name = NULL;
char* tb = NULL;
int32_t numOfRows = 0;
char n[TSDB_TABLE_NAME_LEN] = {0};
while ((name = metaTbCursorNext(pInfo->pCur)) != NULL) {
STR_TO_VARSTR(n, name);
while ((tb = metaTbCursorNext(pInfo->pCur)) != NULL) {
STR_TO_VARSTR(n, tb);
colDataAppend(pTableNameCol, numOfRows, n, false);
numOfRows += 1;
if (numOfRows >= pInfo->capacity) {
@ -992,3 +980,167 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataB
return pOperator;
}
static SSDataBlock* doTagScan(SOperatorInfo* pOperator, bool* newgroup) {
#if 0
SOperatorInfo* pOperator = (SOperatorInfo*) param;
if (pOperator->status == OP_EXEC_DONE) {
return NULL;
}
int32_t maxNumOfTables = (int32_t)pResultInfo->capacity;
STagScanInfo *pInfo = pOperator->info;
SSDataBlock *pRes = pInfo->pRes;
*newgroup = false;
int32_t count = 0;
SArray* pa = GET_TABLEGROUP(pRuntimeEnv, 0);
int32_t functionId = getExprFunctionId(&pOperator->pExpr[0]);
if (functionId == FUNCTION_TID_TAG) { // return the tags & table Id
assert(pQueryAttr->numOfOutput == 1);
SExprInfo* pExprInfo = &pOperator->pExpr[0];
int32_t rsize = pExprInfo->base.resSchema.bytes;
count = 0;
int16_t bytes = pExprInfo->base.resSchema.bytes;
int16_t type = pExprInfo->base.resSchema.type;
for(int32_t i = 0; i < pQueryAttr->numOfTags; ++i) {
if (pQueryAttr->tagColList[i].colId == pExprInfo->base.pColumns->info.colId) {
bytes = pQueryAttr->tagColList[i].bytes;
type = pQueryAttr->tagColList[i].type;
break;
}
}
SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0);
while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) {
int32_t i = pInfo->curPos++;
STableQueryInfo *item = taosArrayGetP(pa, i);
char *output = pColInfo->pData + count * rsize;
varDataSetLen(output, rsize - VARSTR_HEADER_SIZE);
output = varDataVal(output);
STableId* id = TSDB_TABLEID(item->pTable);
*(int16_t *)output = 0;
output += sizeof(int16_t);
*(int64_t *)output = id->uid; // memory align problem, todo serialize
output += sizeof(id->uid);
*(int32_t *)output = id->tid;
output += sizeof(id->tid);
*(int32_t *)output = pQueryAttr->vgId;
output += sizeof(pQueryAttr->vgId);
char* data = NULL;
if (pExprInfo->base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX) {
data = tsdbGetTableName(item->pTable);
} else {
data = tsdbGetTableTagVal(item->pTable, pExprInfo->base.pColumns->info.colId, type, bytes);
}
doSetTagValueToResultBuf(output, data, type, bytes);
count += 1;
}
//qDebug("QInfo:0x%"PRIx64" create (tableId, tag) info completed, rows:%d", GET_TASKID(pRuntimeEnv), count);
} else if (functionId == FUNCTION_COUNT) {// handle the "count(tbname)" query
SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0);
*(int64_t*)pColInfo->pData = pInfo->totalTables;
count = 1;
pOperator->status = OP_EXEC_DONE;
//qDebug("QInfo:0x%"PRIx64" create count(tbname) query, res:%d rows:1", GET_TASKID(pRuntimeEnv), count);
} else { // return only the tags|table name etc.
SExprInfo* pExprInfo = &pOperator->pExpr[0]; // todo use the column list instead of exprinfo
count = 0;
while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) {
int32_t i = pInfo->curPos++;
STableQueryInfo* item = taosArrayGetP(pa, i);
char *data = NULL, *dst = NULL;
int16_t type = 0, bytes = 0;
for(int32_t j = 0; j < pOperator->numOfOutput; ++j) {
// not assign value in case of user defined constant output column
if (TSDB_COL_IS_UD_COL(pExprInfo[j].base.pColumns->flag)) {
continue;
}
SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, j);
type = pExprInfo[j].base.resSchema.type;
bytes = pExprInfo[j].base.resSchema.bytes;
if (pExprInfo[j].base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX) {
data = tsdbGetTableName(item->pTable);
} else {
data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.pColumns->info.colId, type, bytes);
}
dst = pColInfo->pData + count * pExprInfo[j].base.resSchema.bytes;
doSetTagValueToResultBuf(dst, data, type, bytes);
}
count += 1;
}
if (pInfo->curPos >= pInfo->totalTables) {
pOperator->status = OP_EXEC_DONE;
}
//qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_TASKID(pRuntimeEnv), count);
}
if (pOperator->status == OP_EXEC_DONE) {
setTaskStatus(pOperator->pRuntimeEnv, TASK_COMPLETED);
}
pRes->info.rows = count;
return (pRes->info.rows == 0)? NULL:pInfo->pRes;
#endif
return TSDB_CODE_SUCCESS;
}
static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) {
STagScanInfo* pInfo = (STagScanInfo*)param;
pInfo->pRes = blockDataDestroy(pInfo->pRes);
}
SOperatorInfo* createTagScanOperatorInfo(void* pReaderHandle, SExprInfo* pExpr, int32_t numOfOutput, SExecTaskInfo* pTaskInfo) {
STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
goto _error;
}
pInfo->pReader = pReaderHandle;
pInfo->curPos = 0;
pOperator->name = "TagScanOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN;
pOperator->blockingOptr = false;
pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo;
pOperator->getNextFn = doTagScan;
pOperator->pExpr = pExpr;
pOperator->numOfOutput = numOfOutput;
pOperator->pTaskInfo = pTaskInfo;
pOperator->closeFn = destroyTagScanOperatorInfo;
return pOperator;
_error:
taosMemoryFree(pInfo);
taosMemoryFree(pOperator);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}

10
source/libs/function/inc/builtinsimpl.h
View File

@ -24,7 +24,7 @@ extern "C" {
#include "functionMgt.h"
bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId);
int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
EFuncDataRequired countDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow);
bool getCountFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
@ -43,17 +43,17 @@ int32_t maxFunction(SqlFunctionCtx *pCtx);
bool getAvgFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool avgFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
int32_t avgFunction(SqlFunctionCtx* pCtx);
int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId);
int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
bool getStddevFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool stddevFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
int32_t stddevFunction(SqlFunctionCtx* pCtx);
int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId);
int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
bool getPercentileFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool percentileFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
int32_t percentileFunction(SqlFunctionCtx *pCtx);
int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId);
int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
bool getDiffFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool diffFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo);
@ -65,7 +65,7 @@ int32_t lastFunction(SqlFunctionCtx *pCtx);
bool getTopBotFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv);
int32_t topFunction(SqlFunctionCtx *pCtx);
int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId);
int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
#ifdef __cplusplus
}

21
source/libs/function/inc/tudf.h
View File

@ -29,29 +29,20 @@ extern "C" {
#define UDF_LISTEN_PIPE_NAME_LEN 32
#define UDF_LISTEN_PIPE_NAME_PREFIX "udfd.sock."
#define UDF_DNODE_ID_ENV_NAME "DNODE_ID"
//======================================================================================
//begin API to taosd and qworker
enum {
UDFC_CODE_STOPPING = -1,
UDFC_CODE_PIPE_READ_ERR = -3,
UDFC_CODE_PIPE_READ_ERR = -2,
UDFC_CODE_CONNECT_PIPE_ERR = -3,
UDFC_CODE_LOAD_UDF_FAILURE = -4,
UDFC_CODE_INVALID_STATE = -5
};
typedef void *UdfcHandle;
typedef void *UdfcFuncHandle;
/**
* create udfd proxy, called once in process that call setupUdf/callUdfxxx/teardownUdf
* @return error code
*/
int32_t udfcOpen(int32_t dnodeId, UdfcHandle* proxyHandle);
/**
* destroy udfd proxy
* @return error code
*/
int32_t udfcClose(UdfcHandle proxyhandle);
/**
@ -60,7 +51,7 @@ int32_t udfcClose(UdfcHandle proxyhandle);
* @param handle, out
* @return error code
*/
int32_t setupUdf(UdfcHandle proxyHandle, char udfName[], SEpSet *epSet, UdfcFuncHandle *handle);
int32_t setupUdf(char udfName[], UdfcFuncHandle *handle);
typedef struct SUdfColumnMeta {
int16_t type;

2
source/libs/function/inc/tudfInt.h
View File

@ -39,7 +39,6 @@ enum {
typedef struct SUdfSetupRequest {
char udfName[TSDB_FUNC_NAME_LEN];
SEpSet epSet;
} SUdfSetupRequest;
typedef struct SUdfSetupResponse {
@ -112,6 +111,7 @@ void freeUdfDataDataBlock(SUdfDataBlock *block);
int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlock);
int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block);
int32_t getUdfdPipeName(char* pipeName, int32_t size);
#ifdef __cplusplus
}
#endif

2
source/libs/function/src/builtins.c
View File

@ -355,7 +355,7 @@ static int32_t translateToIso8601(SFunctionNode* pFunc, char* pErrBuf, int32_t l
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
pFunc->node.resType = (SDataType) { .bytes = 64, .type = TSDB_DATA_TYPE_BINARY};
pFunc->node.resType = (SDataType) { .bytes = 24, .type = TSDB_DATA_TYPE_BINARY};
return TSDB_CODE_SUCCESS;
}

232
source/libs/function/src/builtinsimpl.c
View File

@ -14,7 +14,7 @@
*/
#include "builtinsimpl.h"
#include <libs/nodes/querynodes.h>
#include "function.h"
#include "querynodes.h"
#include "taggfunction.h"
#include "tdatablock.h"
@ -44,8 +44,6 @@ typedef struct STopBotResItem {
} STopBotResItem;
typedef struct STopBotRes {
int32_t pageId;
// int32_t num;
STopBotResItem *pItems;
} STopBotRes;
@ -92,18 +90,6 @@ typedef struct SDiffInfo {
} \
} while (0);
#define DO_UPDATE_SUBSID_RES(ctx, ts) \
do { \
for (int32_t _i = 0; _i < (ctx)->subsidiaryRes.numOfCols; ++_i) { \
SqlFunctionCtx *__ctx = (ctx)->subsidiaryRes.pCtx[_i]; \
if (__ctx->functionId == FUNCTION_TS_DUMMY) { \
__ctx->tag.i = (ts); \
__ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \
} \
__ctx->fpSet.process(__ctx); \
} \
} while (0)
#define UPDATE_DATA(ctx, left, right, num, sign, _ts) \
do { \
if (((left) < (right)) ^ (sign)) { \
@ -139,7 +125,8 @@ bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) {
return true;
}
int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) {
int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
int32_t slotId = pCtx->pExpr->base.resSchema.slotId;
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
@ -406,7 +393,7 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) {
return TSDB_CODE_SUCCESS;
}
int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) {
int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
SInputColumnInfoData* pInput = &pCtx->input;
int32_t type = pInput->pData[0]->info.type;
SAvgRes* pAvgRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
@ -416,7 +403,7 @@ int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) {
pAvgRes->result = pAvgRes->sum.dsum / ((double) pAvgRes->count);
}
return functionFinalize(pCtx, pBlock, slotId);
return functionFinalize(pCtx, pBlock);
}
EFuncDataRequired statisDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow){
@ -521,6 +508,49 @@ bool getMinmaxFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
return true;
}
#define GET_TS_LIST(x) ((TSKEY*)((x)->ptsList))
#define GET_TS_DATA(x, y) (GET_TS_LIST(x)[(y)])
#define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx) \
do { \
for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \
SqlFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \
__ctx->fpSet.process(__ctx); \
} \
} while (0);
#define DO_UPDATE_SUBSID_RES(ctx, ts) \
do { \
for (int32_t _i = 0; _i < (ctx)->subsidiaries.num; ++_i) { \
SqlFunctionCtx* __ctx = (ctx)->subsidiaries.pCtx[_i]; \
if (__ctx->functionId == FUNCTION_TS_DUMMY) { \
__ctx->tag.i = (ts); \
__ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \
} \
__ctx->fpSet.process(__ctx); \
} \
} while (0)
#define UPDATE_DATA(ctx, left, right, num, sign, _ts) \
do { \
if (((left) < (right)) ^ (sign)) { \
(left) = (right); \
DO_UPDATE_SUBSID_RES(ctx, _ts); \
(num) += 1; \
} \
} while (0)
#define LOOPCHECK_N(val, _col, ctx, _t, _nrow, _start, sign, num) \
do { \
_t *d = (_t *)((_col)->pData); \
for (int32_t i = (_start); i < (_nrow) + (_start); ++i) { \
if (((_col)->hasNull) && colDataIsNull_f((_col)->nullbitmap, i)) { \
continue; \
} \
TSKEY ts = (ctx)->ptsList != NULL ? GET_TS_DATA(ctx, i) : 0; \
UPDATE_DATA(ctx, val, d[i], num, sign, ts); \
} \
} while (0)
int32_t doMinMaxHelper(SqlFunctionCtx *pCtx, int32_t isMinFunc) {
int32_t numOfElems = 0;
@ -564,8 +594,8 @@ int32_t doMinMaxHelper(SqlFunctionCtx *pCtx, int32_t isMinFunc) {
int64_t val = GET_INT64_VAL(tval);
if ((prev < val) ^ isMinFunc) {
*(int64_t*) buf = val;
for (int32_t i = 0; i < (pCtx)->subsidiaryRes.numOfCols; ++i) {
SqlFunctionCtx* __ctx = pCtx->subsidiaryRes.pCtx[i];
for (int32_t i = 0; i < (pCtx)->subsidiaries.num; ++i) {
SqlFunctionCtx* __ctx = pCtx->subsidiaries.pCtx[i];
if (__ctx->functionId == FUNCTION_TS_DUMMY) { // TODO refactor
__ctx->tag.i = key;
__ctx->tag.nType = TSDB_DATA_TYPE_BIGINT;
@ -581,8 +611,8 @@ int32_t doMinMaxHelper(SqlFunctionCtx *pCtx, int32_t isMinFunc) {
uint64_t val = GET_UINT64_VAL(tval);
if ((prev < val) ^ isMinFunc) {
*(uint64_t*) buf = val;
for (int32_t i = 0; i < (pCtx)->subsidiaryRes.numOfCols; ++i) {
SqlFunctionCtx* __ctx = pCtx->subsidiaryRes.pCtx[i];
for (int32_t i = 0; i < (pCtx)->subsidiaries.num; ++i) {
SqlFunctionCtx* __ctx = pCtx->subsidiaries.pCtx[i];
if (__ctx->functionId == FUNCTION_TS_DUMMY) { // TODO refactor
__ctx->tag.i = key;
__ctx->tag.nType = TSDB_DATA_TYPE_BIGINT;
@ -797,7 +827,7 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx) {
return TSDB_CODE_SUCCESS;
}
int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) {
int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
SInputColumnInfoData* pInput = &pCtx->input;
int32_t type = pInput->pData[0]->info.type;
SStddevRes* pStddevRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
@ -810,7 +840,7 @@ int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId
pStddevRes->result = sqrt(pStddevRes->quadraticDSum/((double)pStddevRes->count) - avg*avg);
}
return functionFinalize(pCtx, pBlock, slotId);
return functionFinalize(pCtx, pBlock);
}
bool getPercentileFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
@ -923,7 +953,7 @@ int32_t percentileFunction(SqlFunctionCtx *pCtx) {
return TSDB_CODE_SUCCESS;
}
int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) {
int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
SVariant* pVal = &pCtx->param[1].param;
double v = pVal->nType == TSDB_DATA_TYPE_INT ? pVal->i : pVal->d;
@ -936,7 +966,7 @@ int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t sl
}
tMemBucketDestroy(pMemBucket);
return functionFinalize(pCtx, pBlock, slotId);
return functionFinalize(pCtx, pBlock);
}
bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
@ -1353,15 +1383,16 @@ static STopBotRes *getTopBotOutputInfo(SqlFunctionCtx *pCtx) {
return pRes;
}
static void doAddIntoResult(STopBotRes* pRes, int32_t maxSize, void* pData, int32_t rowIndex, SSDataBlock* pSrcBlock,
static void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSDataBlock* pSrcBlock,
uint16_t type, uint64_t uid, SResultRowEntryInfo* pEntryInfo);
static void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STopBotResItem* pItem);
static void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STopBotResItem* pItem);
int32_t topFunction(SqlFunctionCtx *pCtx) {
int32_t numOfElems = 0;
SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx);
STopBotRes *pRes = getTopBotOutputInfo(pCtx);
// if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotRes) + POINTER_BYTES * pCtx->param[0].i)) {
// buildTopBotStruct(pRes, pCtx);
// }
@ -1381,7 +1412,7 @@ int32_t topFunction(SqlFunctionCtx *pCtx) {
numOfElems++;
char* data = colDataGetData(pCol, i);
doAddIntoResult(pRes, pCtx->param[1].param.i, data, i, NULL, type, pInput->uid, pResInfo);
doAddIntoResult(pCtx, data, i, pCtx->pSrcBlock, type, pInput->uid, pResInfo);
}
return TSDB_CODE_SUCCESS;
@ -1414,9 +1445,11 @@ static int32_t topBotResComparFn(const void *p1, const void *p2, const void *par
return (val1->v.d > val2->v.d) ? 1 : -1;
}
void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSDataBlock* pSrcBlock, uint16_t type,
uint64_t uid, SResultRowEntryInfo* pEntryInfo) {
STopBotRes *pRes = getTopBotOutputInfo(pCtx);
int32_t maxSize = pCtx->param[1].param.i;
void doAddIntoResult(STopBotRes *pRes, int32_t maxSize, void *pData, int32_t rowIndex, SSDataBlock* pSrcBlock, uint16_t type,
uint64_t uid, SResultRowEntryInfo* pEntryInfo) {
SVariant val = {0};
taosVariantCreateFromBinary(&val, pData, tDataTypes[type].bytes, type);
@ -1428,22 +1461,9 @@ void doAddIntoResult(STopBotRes *pRes, int32_t maxSize, void *pData, int32_t row
STopBotResItem* pItem = &pItems[pEntryInfo->numOfRes];
pItem->v = val;
pItem->uid = uid;
pItem->tuplePos.pageId = -1; // todo set the corresponding tuple data in the disk-based buffer
if (pRes->pageId == -1) {
SFilePage* pPage = getNewBufPage(NULL, 0, &pRes->pageId);
pPage->num = sizeof(SFilePage);
// keep the current row data
for(int32_t i = 0; i < pSrcBlock->info.numOfCols; ++i) {
SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i);
bool isNull = colDataIsNull_s(pCol, rowIndex);
colDataGetData(pCol, rowIndex);
}
}
// save the data of this tuple
saveTupleData(pCtx, rowIndex, pSrcBlock, pItem);
// allocate the buffer and keep the data of this row into the new allocated buffer
pEntryInfo->numOfRes++;
@ -1452,22 +1472,100 @@ void doAddIntoResult(STopBotRes *pRes, int32_t maxSize, void *pData, int32_t row
if ((IS_SIGNED_NUMERIC_TYPE(type) && val.i > pItems[0].v.i) ||
(IS_UNSIGNED_NUMERIC_TYPE(type) && val.u > pItems[0].v.u) ||
(IS_FLOAT_TYPE(type) && val.d > pItems[0].v.d)) {
// replace the old data and the coresponding tuple data
STopBotResItem* pItem = &pItems[0];
pItem->v = val;
pItem->uid = uid;
pItem->tuplePos.pageId = -1; // todo set the corresponding tuple data in the disk-based buffer
// save the data of this tuple by over writing the old data
copyTupleData(pCtx, rowIndex, pSrcBlock, pItem);
taosheapadjust((void *) pItems, sizeof(STopBotResItem), 0, pEntryInfo->numOfRes - 1, (const void *) &type, topBotResComparFn, NULL, false);
}
}
}
int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) {
void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STopBotResItem* pItem) {
SFilePage* pPage = NULL;
int32_t completeRowSize = pSrcBlock->info.rowSize + pSrcBlock->info.numOfCols * sizeof(bool);
if (pCtx->curBufPage == -1) {
pPage = getNewBufPage(pCtx->pBuf, 0, &pCtx->curBufPage);
pPage->num = sizeof(SFilePage);
} else {
pPage = getBufPage(pCtx->pBuf, pCtx->curBufPage);
if (pPage->num + completeRowSize > getBufPageSize(pCtx->pBuf)) {
pPage = getNewBufPage(pCtx->pBuf, 0, &pCtx->curBufPage);
pPage->num = sizeof(SFilePage);
}
}
pItem->tuplePos.pageId = pCtx->curBufPage;
// keep the current row data, extract method
int32_t offset = 0;
bool* nullList = (bool*)((char*)pPage + pPage->num);
char* pStart = (char*)(nullList + sizeof(bool) * pSrcBlock->info.numOfCols);
for (int32_t i = 0; i < pSrcBlock->info.numOfCols; ++i) {
SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i);
bool isNull = colDataIsNull_s(pCol, rowIndex);
if (isNull) {
nullList[i] = true;
continue;
}
char* p = colDataGetData(pCol, rowIndex);
if (IS_VAR_DATA_TYPE(pCol->info.type)) {
memcpy(pStart + offset, p, varDataTLen(p));
} else {
memcpy(pStart + offset, p, pCol->info.bytes);
}
offset += pCol->info.bytes;
}
pItem->tuplePos.offset = pPage->num;
pPage->num += completeRowSize;
setBufPageDirty(pPage, true);
releaseBufPage(pCtx->pBuf, pPage);
}
void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STopBotResItem* pItem) {
SFilePage* pPage = getBufPage(pCtx->pBuf, pItem->tuplePos.pageId);
bool* nullList = (bool*)((char*)pPage + pItem->tuplePos.offset);
char* pStart = (char*)(nullList + pSrcBlock->info.numOfCols * sizeof(bool));
int32_t offset = 0;
for(int32_t i = 0; i < pSrcBlock->info.numOfCols; ++i) {
SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i);
if ((nullList[i] = colDataIsNull_s(pCol, rowIndex)) == true) {
continue;
}
char* p = colDataGetData(pCol, rowIndex);
if (IS_VAR_DATA_TYPE(pCol->info.type)) {
memcpy(pStart + offset, p, varDataTLen(p));
} else {
memcpy(pStart + offset, p, pCol->info.bytes);
}
offset += pCol->info.bytes;
}
setBufPageDirty(pPage, true);
releaseBufPage(pCtx->pBuf, pPage);
}
int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
SResultRowEntryInfo *pEntryInfo = GET_RES_INFO(pCtx);
STopBotRes* pRes = GET_ROWCELL_INTERBUF(pEntryInfo);
pEntryInfo->complete = true;
int32_t type = pCtx->input.pData[0]->info.type;
int32_t slotId = pCtx->pExpr->base.resSchema.slotId;
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
// todo assign the tag value and the corresponding row data
@ -1476,19 +1574,45 @@ int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId
case TSDB_DATA_TYPE_INT: {
for (int32_t i = 0; i < pEntryInfo->numOfRes; ++i) {
STopBotResItem* pItem = &pRes->pItems[i];
colDataAppendInt32(pCol, currentRow++, (int32_t*)&pItem->v.i);
colDataAppendInt32(pCol, currentRow, (int32_t*)&pItem->v.i);
int32_t pageId = pItem->tuplePos.pageId;
int32_t offset = pItem->tuplePos.offset;
if (pageId != -1) {
// todo
if (pItem->tuplePos.pageId != -1) {
SFilePage* pPage = getBufPage(pCtx->pBuf, pageId);
bool* nullList = (bool*)((char*)pPage + offset);
char* pStart = (char*)(nullList + pCtx->pSrcBlock->info.numOfCols * sizeof(bool));
// todo set the offset value to optimize the performance.
for (int32_t j = 0; j < pCtx->subsidiaries.num; ++j) {
SqlFunctionCtx* pc = pCtx->subsidiaries.pCtx[j];
SFunctParam *pFuncParam = &pc->pExpr->base.pParam[0];
int32_t srcSlotId = pFuncParam->pCol->slotId;
int32_t dstSlotId = pCtx->pExpr->base.resSchema.slotId;
int32_t ps = 0;
for(int32_t k = 0; k < srcSlotId; ++k) {
SColumnInfoData* pSrcCol = taosArrayGet(pCtx->pSrcBlock->pDataBlock, k);
ps += pSrcCol->info.bytes;
}
SColumnInfoData* pDstCol = taosArrayGet(pBlock->pDataBlock, dstSlotId);
if (nullList[srcSlotId]) {
colDataAppendNULL(pDstCol, currentRow);
} else {
colDataAppend(pDstCol, currentRow, (pStart + ps), false);
}
}
}
currentRow += 1;
}
break;
}
}
return pEntryInfo->numOfRes;
// return functionFinalize(pCtx, pBlock, slotId);
}

161
source/libs/function/src/tudf.c
View File

@ -124,7 +124,7 @@ enum {
int64_t gUdfTaskSeqNum = 0;
typedef struct SUdfdProxy {
int32_t dnodeId;
char udfdPipeName[UDF_LISTEN_PIPE_NAME_LEN];
uv_barrier_t gUdfInitBarrier;
uv_loop_t gUdfdLoop;
@ -137,11 +137,11 @@ typedef struct SUdfdProxy {
int8_t gUdfcState;
QUEUE gUdfTaskQueue;
QUEUE gUvProcTaskQueue;
// int8_t gUdfcState = UDFC_STATE_INITAL;
// QUEUE gUdfTaskQueue = {0};
// QUEUE gUvProcTaskQueue = {0};
int8_t initialized;
} SUdfdProxy;
SUdfdProxy gUdfdProxy = {0};
typedef struct SUdfUvSession {
SUdfdProxy *udfc;
@ -209,19 +209,27 @@ enum {
UDFC_STATE_STARTNG, // starting after udfcOpen
UDFC_STATE_READY, // started and begin to receive quests
UDFC_STATE_STOPPING, // stopping after udfcClose
UDFC_STATUS_FINAL, // stopped
};
int32_t getUdfdPipeName(char* pipeName, int32_t size) {
char dnodeId[8] = {0};
size_t dnodeIdSize;
int32_t err = uv_os_getenv(UDF_DNODE_ID_ENV_NAME, dnodeId, &dnodeIdSize);
if (err != 0) {
dnodeId[0] = '1';
}
snprintf(pipeName, size, "%s%s", UDF_LISTEN_PIPE_NAME_PREFIX, dnodeId);
return 0;
}
int32_t encodeUdfSetupRequest(void **buf, const SUdfSetupRequest *setup) {
int32_t len = 0;
len += taosEncodeBinary(buf, setup->udfName, TSDB_FUNC_NAME_LEN);
len += taosEncodeSEpSet(buf, &setup->epSet);
return len;
}
void* decodeUdfSetupRequest(const void* buf, SUdfSetupRequest *request) {
buf = taosDecodeBinaryTo(buf, request->udfName, TSDB_FUNC_NAME_LEN);
buf = taosDecodeSEpSet((void*)buf, &request->epSet);
return (void*)buf;
}
@ -604,7 +612,7 @@ void onUdfcPipeClose(uv_handle_t *handle) {
}
int32_t udfcGetUvTaskResponseResult(SClientUdfTask *task, SClientUvTaskNode *uvTask) {
debugPrint("%s", "get uv task result");
fnDebug("udfc get uv task result. task: %p", task);
if (uvTask->type == UV_TASK_REQ_RSP) {
if (uvTask->rspBuf.base != NULL) {
SUdfResponse rsp;
@ -647,7 +655,6 @@ int32_t udfcGetUvTaskResponseResult(SClientUdfTask *task, SClientUvTaskNode *uvT
}
void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) {
debugPrint("%s", "client allocate buffer to receive from pipe");
SClientUvConn *conn = handle->data;
SClientConnBuf *connBuf = &conn->readBuf;
@ -662,7 +669,7 @@ void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf
buf->base = connBuf->buf;
buf->len = connBuf->cap;
} else {
//TODO: log error
fnError("udfc allocate buffer failure. size: %d", msgHeadSize);
buf->base = NULL;
buf->len = 0;
}
@ -674,13 +681,13 @@ void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf
buf->base = connBuf->buf + connBuf->len;
buf->len = connBuf->cap - connBuf->len;
} else {
//TODO: log error free connBuf->buf
fnError("udfc re-allocate buffer failure. size: %d", connBuf->cap);
buf->base = NULL;
buf->len = 0;
}
}
debugPrint("\tconn buf cap - len - total : %d - %d - %d", connBuf->cap, connBuf->len, connBuf->total);
fnTrace("conn buf cap - len - total : %d - %d - %d", connBuf->cap, connBuf->len, connBuf->total);
}
@ -689,6 +696,7 @@ bool isUdfcUvMsgComplete(SClientConnBuf *connBuf) {
connBuf->total = *(int32_t *) (connBuf->buf);
}
if (connBuf->len == connBuf->cap && connBuf->total == connBuf->cap) {
fnTrace("udfc complete message is received, now handle it");
return true;
}
return false;
@ -696,10 +704,10 @@ bool isUdfcUvMsgComplete(SClientConnBuf *connBuf) {
void udfcUvHandleRsp(SClientUvConn *conn) {
SClientConnBuf *connBuf = &conn->readBuf;
int64_t seqNum = *(int64_t *) (connBuf->buf + sizeof(int32_t)); // msglen int32_t then seqnum
int64_t seqNum = *(int64_t *) (connBuf->buf + sizeof(int32_t)); // msglen then seqnum
if (QUEUE_EMPTY(&conn->taskQueue)) {
//LOG error
fnError("udfc no task waiting for response on connection");
return;
}
bool found = false;
@ -713,7 +721,7 @@ void udfcUvHandleRsp(SClientUvConn *conn) {
found = true;
taskFound = task;
} else {
//LOG error;
fnError("udfc more than one task waiting for the same response");
continue;
}
}
@ -727,7 +735,7 @@ void udfcUvHandleRsp(SClientUvConn *conn) {
uv_sem_post(&taskFound->taskSem);
QUEUE_REMOVE(&taskFound->procTaskQueue);
} else {
//TODO: LOG error
fnError("no task is waiting for the response.");
}
connBuf->buf = NULL;
connBuf->total = -1;
@ -751,7 +759,7 @@ void udfcUvHandleError(SClientUvConn *conn) {
}
void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) {
debugPrint("%s, nread: %zd", "client read from pipe", nread);
fnTrace("udfc client %p, client read from pipe. nread: %zd", client, nread);
if (nread == 0) return;
SClientUvConn *conn = client->data;
@ -764,9 +772,9 @@ void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) {
}
if (nread < 0) {
debugPrint("\tclient read error: %s", uv_strerror(nread));
fnError("udfc client pipe %p read error: %s", client, uv_strerror(nread));
if (nread == UV_EOF) {
//TODO:
fnError("udfc client pipe %p closed", client);
}
udfcUvHandleError(conn);
}
@ -774,16 +782,15 @@ void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) {
}
void onUdfClientWrite(uv_write_t *write, int status) {
debugPrint("%s", "after writing to pipe");
SClientUvTaskNode *uvTask = write->data;
uv_pipe_t *pipe = uvTask->pipe;
if (status == 0) {
uv_pipe_t *pipe = uvTask->pipe;
SClientUvConn *conn = pipe->data;
QUEUE_INSERT_TAIL(&conn->taskQueue, &uvTask->connTaskQueue);
} else {
//TODO Log error;
fnError("udfc client %p write error.", pipe);
}
debugPrint("\tlength:%zu", uvTask->reqBuf.len);
fnTrace("udfc client %p write length:%zu", pipe, uvTask->reqBuf.len);
taosMemoryFree(write);
taosMemoryFree(uvTask->reqBuf.base);
}
@ -841,7 +848,7 @@ int32_t createUdfcUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskN
}
int32_t queueUvUdfTask(SClientUvTaskNode *uvTask) {
debugPrint("%s, %d", "queue uv task", uvTask->type);
fnTrace("queue uv task to event loop, task: %d, %p", uvTask->type, uvTask);
SUdfdProxy *udfc = uvTask->udfc;
uv_mutex_lock(&udfc->gUdfTaskQueueMutex);
QUEUE_INSERT_TAIL(&udfc->gUdfTaskQueue, &uvTask->recvTaskQueue);
@ -855,7 +862,7 @@ int32_t queueUvUdfTask(SClientUvTaskNode *uvTask) {
}
int32_t startUvUdfTask(SClientUvTaskNode *uvTask) {
debugPrint("%s, type %d", "start uv task ", uvTask->type);
fnTrace("event loop start uv task. task: %d, %p", uvTask->type, uvTask);
switch (uvTask->type) {
case UV_TASK_CONNECT: {
uv_pipe_t *pipe = taosMemoryMalloc(sizeof(uv_pipe_t));
@ -874,8 +881,7 @@ int32_t startUvUdfTask(SClientUvTaskNode *uvTask) {
uv_connect_t *connReq = taosMemoryMalloc(sizeof(uv_connect_t));
connReq->data = uvTask;
uv_pipe_connect(connReq, pipe, "udf.sock", onUdfClientConnect);
uv_pipe_connect(connReq, pipe, uvTask->udfc->udfdPipeName, onUdfClientConnect);
break;
}
case UV_TASK_REQ_RSP: {
@ -971,27 +977,37 @@ void constructUdfService(void *argsThread) {
uv_loop_close(&udfc->gUdfdLoop);
}
int32_t udfcOpen(int32_t dnodeId, UdfcHandle *udfc) {
SUdfdProxy *proxy = taosMemoryCalloc(1, sizeof(SUdfdProxy));
proxy->dnodeId = dnodeId;
int32_t udfcOpen() {
int8_t old = atomic_val_compare_exchange_8(&gUdfdProxy.initialized, 0, 1);
if (old == 1) {
return 0;
}
SUdfdProxy *proxy = &gUdfdProxy;
getUdfdPipeName(proxy->udfdPipeName, UDF_LISTEN_PIPE_NAME_LEN);
proxy->gUdfcState = UDFC_STATE_STARTNG;
uv_barrier_init(&proxy->gUdfInitBarrier, 2);
uv_thread_create(&proxy->gUdfLoopThread, constructUdfService, proxy);
uv_barrier_wait(&proxy->gUdfInitBarrier);
atomic_store_8(&proxy->gUdfcState, UDFC_STATE_READY);
proxy->gUdfcState = UDFC_STATE_READY;
*udfc = proxy;
uv_barrier_wait(&proxy->gUdfInitBarrier);
fnInfo("udfc initialized")
return 0;
}
int32_t udfcClose(UdfcHandle udfcHandle) {
SUdfdProxy *udfc = udfcHandle;
int32_t udfcClose() {
int8_t old = atomic_val_compare_exchange_8(&gUdfdProxy.initialized, 1, 0);
if (old == 0) {
return 0;
}
SUdfdProxy *udfc = &gUdfdProxy;
udfc->gUdfcState = UDFC_STATE_STOPPING;
uv_async_send(&udfc->gUdfLoopStopAsync);
uv_thread_join(&udfc->gUdfLoopThread);
uv_mutex_destroy(&udfc->gUdfTaskQueueMutex);
uv_barrier_destroy(&udfc->gUdfInitBarrier);
udfc->gUdfcState = UDFC_STATUS_FINAL;
taosMemoryFree(udfc);
udfc->gUdfcState = UDFC_STATE_INITAL;
fnInfo("udfc cleaned up");
return 0;
}
@ -1009,12 +1025,15 @@ int32_t udfcRunUvTask(SClientUdfTask *task, int8_t uvTaskType) {
return task->errCode;
}
int32_t setupUdf(UdfcHandle udfc, char udfName[], SEpSet *epSet, UdfcFuncHandle *funcHandle) {
debugPrint("%s", "client setup udf");
int32_t setupUdf(char udfName[], UdfcFuncHandle *funcHandle) {
fnInfo("udfc setup udf. udfName: %s", udfName);
if (gUdfdProxy.gUdfcState != UDFC_STATE_READY) {
return UDFC_CODE_INVALID_STATE;
}
SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask));
task->errCode = 0;
task->session = taosMemoryMalloc(sizeof(SUdfUvSession));
task->session->udfc = udfc;
task->session->udfc = &gUdfdProxy;
task->type = UDF_TASK_SETUP;
SUdfSetupRequest *req = &task->_setup.req;
@ -1022,15 +1041,20 @@ int32_t setupUdf(UdfcHandle udfc, char udfName[], SEpSet *epSet, UdfcFuncHandle
int32_t errCode = udfcRunUvTask(task, UV_TASK_CONNECT);
if (errCode != 0) {
//TODO: log error
return -1;
fnError("failed to connect to pipe. udfName: %s, pipe: %s", udfName, (&gUdfdProxy)->udfdPipeName);
return UDFC_CODE_CONNECT_PIPE_ERR;
}
udfcRunUvTask(task, UV_TASK_REQ_RSP);
SUdfSetupResponse *rsp = &task->_setup.rsp;
task->session->severHandle = rsp->udfHandle;
*funcHandle = task->session;
if (task->errCode != 0) {
fnError("failed to setup udf. err: %d", task->errCode)
} else {
fnInfo("sucessfully setup udf func handle. handle: %p", task->session);
*funcHandle = task->session;
}
int32_t err = task->errCode;
taosMemoryFree(task);
return err;
@ -1038,7 +1062,7 @@ int32_t setupUdf(UdfcHandle udfc, char udfName[], SEpSet *epSet, UdfcFuncHandle
int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdfInterBuf *state, SUdfInterBuf *state2,
SSDataBlock* output, SUdfInterBuf *newState) {
debugPrint("%s", "client call udf");
fnTrace("udfc call udf. callType: %d, funcHandle: %p", callType, handle);
SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask));
task->errCode = 0;
@ -1076,35 +1100,37 @@ int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdf
udfcRunUvTask(task, UV_TASK_REQ_RSP);
SUdfCallResponse *rsp = &task->_call.rsp;
switch (callType) {
case TSDB_UDF_CALL_AGG_INIT: {
*newState = rsp->resultBuf;
break;
}
case TSDB_UDF_CALL_AGG_PROC: {
*newState = rsp->resultBuf;
break;
}
case TSDB_UDF_CALL_AGG_MERGE: {
*newState = rsp->resultBuf;
break;
}
case TSDB_UDF_CALL_AGG_FIN: {
*newState = rsp->resultBuf;
break;
}
case TSDB_UDF_CALL_SCALA_PROC: {
*output = rsp->resultData;
break;
if (task->errCode != 0) {
fnError("call udf failure. err: %d", task->errCode);
} else {
SUdfCallResponse *rsp = &task->_call.rsp;
switch (callType) {
case TSDB_UDF_CALL_AGG_INIT: {
*newState = rsp->resultBuf;
break;
}
case TSDB_UDF_CALL_AGG_PROC: {
*newState = rsp->resultBuf;
break;
}
case TSDB_UDF_CALL_AGG_MERGE: {
*newState = rsp->resultBuf;
break;
}
case TSDB_UDF_CALL_AGG_FIN: {
*newState = rsp->resultBuf;
break;
}
case TSDB_UDF_CALL_SCALA_PROC: {
*output = rsp->resultData;
break;
}
}
}
taosMemoryFree(task);
return task->errCode;
}
//TODO: translate these calls to callUdf
int32_t callUdfAggInit(UdfcFuncHandle handle, SUdfInterBuf *interBuf) {
int8_t callType = TSDB_UDF_CALL_AGG_INIT;
@ -1148,7 +1174,7 @@ int32_t callUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t nu
}
int32_t teardownUdf(UdfcFuncHandle handle) {
debugPrint("%s", "client teardown udf");
fnInfo("tear down udf. udf func handle: %p", handle);
SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask));
task->errCode = 0;
@ -1160,7 +1186,6 @@ int32_t teardownUdf(UdfcFuncHandle handle) {
udfcRunUvTask(task, UV_TASK_REQ_RSP);
SUdfTeardownResponse *rsp = &task->_teardown.rsp;
int32_t err = task->errCode;

103
source/libs/function/src/udfd.c
View File

@ -20,6 +20,7 @@
#include "tudf.h"
#include "tudfInt.h"
#include "tdatablock.h"
#include "tdataformat.h"
#include "tglobal.h"
#include "tmsg.h"
@ -31,8 +32,9 @@ typedef struct SUdfdContext {
uv_signal_t intrSignal;
char listenPipeName[UDF_LISTEN_PIPE_NAME_LEN];
uv_pipe_t listeningPipe;
void *clientRpc;
void *clientRpc;
SCorEpSet mgmtEp;
uv_mutex_t udfsMutex;
SHashObj *udfsHash;
@ -63,8 +65,13 @@ typedef struct SUdf {
uv_mutex_t lock;
uv_cond_t condReady;
char name[16];
int8_t type;
char name[TSDB_FUNC_NAME_LEN];
int8_t funcType;
int8_t scriptType;
int8_t outputType;
int32_t outputLen;
int32_t bufSize;
char path[PATH_MAX];
uv_lib_t lib;
@ -78,17 +85,17 @@ typedef struct SUdfcFuncHandle {
SUdf *udf;
} SUdfcFuncHandle;
int32_t udfdFillUdfInfoFromMNode(void *clientRpc, SEpSet *pEpSet, char *udfName, SUdf *udf);
int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf);
int32_t udfdLoadUdf(char *udfName, SEpSet *pEpSet, SUdf *udf) {
int32_t udfdLoadUdf(char *udfName, SUdf *udf) {
strcpy(udf->name, udfName);
udfdFillUdfInfoFromMNode(global.clientRpc, pEpSet, udf->name, udf);
udfdFillUdfInfoFromMNode(global.clientRpc, udf->name, udf);
//strcpy(udf->path, "/home/slzhou/TDengine/debug/build/lib/libudf1.so");
int err = uv_dlopen(udf->path, &udf->lib);
if (err != 0) {
fnError("can not load library %s. error: %s", udf->path, uv_strerror(err));
// TODO set error
return UDFC_CODE_LOAD_UDF_FAILURE;
}
// TODO: find all the functions
char normalFuncName[TSDB_FUNC_NAME_LEN] = {0};
@ -115,8 +122,8 @@ void udfdProcessRequest(uv_work_t *req) {
SUdf *udf = NULL;
uv_mutex_lock(&global.udfsMutex);
SUdf **udfInHash = taosHashGet(global.udfsHash, request.setup.udfName, TSDB_FUNC_NAME_LEN);
if (*udfInHash) {
SUdf **udfInHash = taosHashGet(global.udfsHash, request.setup.udfName, strlen(request.setup.udfName));
if (udfInHash) {
++(*udfInHash)->refCount;
udf = *udfInHash;
uv_mutex_unlock(&global.udfsMutex);
@ -128,14 +135,14 @@ void udfdProcessRequest(uv_work_t *req) {
uv_mutex_init(&udfNew->lock);
uv_cond_init(&udfNew->condReady);
udf = udfNew;
taosHashPut(global.udfsHash, request.setup.udfName, TSDB_FUNC_NAME_LEN, &udfNew, sizeof(&udfNew));
taosHashPut(global.udfsHash, request.setup.udfName, strlen(request.setup.udfName), &udfNew, sizeof(&udfNew));
uv_mutex_unlock(&global.udfsMutex);
}
uv_mutex_lock(&udf->lock);
if (udf->state == UDF_STATE_INIT) {
udf->state = UDF_STATE_LOADING;
udfdLoadUdf(setup->udfName, &setup->epSet, udf);
udfdLoadUdf(setup->udfName, udf);
udf->state = UDF_STATE_READY;
uv_cond_broadcast(&udf->condReady);
uv_mutex_unlock(&udf->lock);
@ -214,7 +221,7 @@ void udfdProcessRequest(uv_work_t *req) {
udf->refCount--;
if (udf->refCount == 0) {
unloadUdf = true;
taosHashRemove(global.udfsHash, udf->name, TSDB_FUNC_NAME_LEN);
taosHashRemove(global.udfsHash, udf->name, strlen(udf->name));
}
uv_mutex_unlock(&global.udfsMutex);
if (unloadUdf) {
@ -393,7 +400,48 @@ void udfdIntrSignalHandler(uv_signal_t *handle, int signum) {
void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { return; }
int32_t udfdFillUdfInfoFromMNode(void *clientRpc, SEpSet *pEpSet, char *udfName, SUdf *udf) {
int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet) {
pEpSet->version = 0;
// init mnode ip set
SEpSet* mgmtEpSet = &(pEpSet->epSet);
mgmtEpSet->numOfEps = 0;
mgmtEpSet->inUse = 0;
if (firstEp && firstEp[0] != 0) {
if (strlen(firstEp) >= TSDB_EP_LEN) {
terrno = TSDB_CODE_TSC_INVALID_FQDN;
return -1;
}
int32_t code = taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[0]);
if (code != TSDB_CODE_SUCCESS) {
terrno = TSDB_CODE_TSC_INVALID_FQDN;
return terrno;
}
mgmtEpSet->numOfEps++;
}
if (secondEp && secondEp[0] != 0) {
if (strlen(secondEp) >= TSDB_EP_LEN) {
terrno = TSDB_CODE_TSC_INVALID_FQDN;
return -1;
}
taosGetFqdnPortFromEp(secondEp, &mgmtEpSet->eps[mgmtEpSet->numOfEps]);
mgmtEpSet->numOfEps++;
}
if (mgmtEpSet->numOfEps == 0) {
terrno = TSDB_CODE_TSC_INVALID_FQDN;
return -1;
}
return 0;
}
int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf) {
SRetrieveFuncReq retrieveReq = {0};
retrieveReq.numOfFuncs = 1;
retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN);
@ -410,15 +458,21 @@ int32_t udfdFillUdfInfoFromMNode(void *clientRpc, SEpSet *pEpSet, char *udfName,
rpcMsg.msgType = TDMT_MND_RETRIEVE_FUNC;
SRpcMsg rpcRsp = {0};
rpcSendRecv(clientRpc, pEpSet, &rpcMsg, &rpcRsp);
rpcSendRecv(clientRpc, &global.mgmtEp.epSet, &rpcMsg, &rpcRsp);
SRetrieveFuncRsp retrieveRsp = {0};
tDeserializeSRetrieveFuncRsp(rpcRsp.pCont, rpcRsp.contLen, &retrieveRsp);
SFuncInfo *pFuncInfo = (SFuncInfo *)taosArrayGet(retrieveRsp.pFuncInfos, 0);
udf->funcType = pFuncInfo->funcType;
udf->scriptType = pFuncInfo->scriptType;
udf->outputType = pFuncInfo->funcType;
udf->outputLen = pFuncInfo->outputLen;
udf->bufSize = pFuncInfo->bufSize;
char path[PATH_MAX] = {0};
taosGetTmpfilePath("/tmp", "libudf", path);
TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);
snprintf(path, sizeof(path), "%s/lib%s.so", "/tmp", udfName);
TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL);
// TODO check for failure of flush to disk
taosWriteFile(file, pFuncInfo->pCode, pFuncInfo->codeSize);
taosCloseFile(&file);
@ -531,15 +585,7 @@ static int32_t udfdUvInit() {
uv_pipe_open(&global.ctrlPipe, 0);
uv_read_start((uv_stream_t *)&global.ctrlPipe, udfdCtrlAllocBufCb, udfdCtrlReadCb);
char dnodeId[8] = {0};
size_t dnodeIdSize;
int32_t err = uv_os_getenv("DNODE_ID", dnodeId, &dnodeIdSize);
if (err != 0) {
dnodeId[0] = '1';
}
char listenPipeName[32] = {0};
snprintf(listenPipeName, sizeof(listenPipeName), "%s%s", UDF_LISTEN_PIPE_NAME_PREFIX, dnodeId);
strcpy(global.listenPipeName, listenPipeName);
getUdfdPipeName(global.listenPipeName, UDF_LISTEN_PIPE_NAME_LEN);
removeListeningPipe();
@ -550,7 +596,7 @@ static int32_t udfdUvInit() {
int r;
fnInfo("bind to pipe %s", global.listenPipeName);
if ((r = uv_pipe_bind(&global.listeningPipe, listenPipeName))) {
if ((r = uv_pipe_bind(&global.listeningPipe, global.listenPipeName))) {
fnError("Bind error %s", uv_err_name(r));
removeListeningPipe();
return -1;
@ -580,7 +626,7 @@ static int32_t udfdRun() {
fnInfo("start the udfd");
int code = uv_run(global.loop, UV_RUN_DEFAULT);
fnInfo("udfd stopped. result: %s", uv_err_name(code));
fnInfo("udfd stopped. result: %s, code: %d", uv_err_name(code), code);
int codeClose = uv_loop_close(global.loop);
fnDebug("uv loop close. result: %s", uv_err_name(codeClose));
udfdCloseClientRpc();
@ -615,5 +661,6 @@ int main(int argc, char *argv[]) {
return -1;
}
initEpSetFromCfg(tsFirst, tsSecond, &global.mgmtEp);
return udfdRun();
}

121
source/libs/function/test/runUdf.c
View File

@ -1,61 +1,84 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "uv.h"
#include "fnLog.h"
#include "os.h"
#include "tudf.h"
#include "tdatablock.h"
#include "tglobal.h"
#include "tudf.h"
static int32_t parseArgs(int32_t argc, char *argv[]) {
for (int32_t i = 1; i < argc; ++i) {
if (strcmp(argv[i], "-c") == 0) {
if (i < argc - 1) {
if (strlen(argv[++i]) >= PATH_MAX) {
printf("config file path overflow");
return -1;
}
tstrncpy(configDir, argv[i], PATH_MAX);
} else {
printf("'-c' requires a parameter, default is %s\n", configDir);
return -1;
}
}
}
return 0;
}
static int32_t initLog() {
char logName[12] = {0};
snprintf(logName, sizeof(logName), "%slog", "udfc");
return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, 0);
}
int main(int argc, char *argv[]) {
UdfcHandle udfc;
udfcOpen(1, &udfc);
uv_sleep(1000);
char path[256] = {0};
size_t cwdSize = 256;
int err = uv_cwd(path, &cwdSize);
if (err != 0) {
fprintf(stderr, "err cwd: %s\n", uv_strerror(err));
return err;
parseArgs(argc, argv);
initLog();
if (taosInitCfg(configDir, NULL, NULL, NULL, 0) != 0) {
fnError("failed to start since read config error");
return -1;
}
udfcOpen();
uv_sleep(1000);
UdfcFuncHandle handle;
setupUdf("udf1", &handle);
SSDataBlock block = {0};
SSDataBlock *pBlock = &block;
pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
pBlock->info.numOfCols = 1;
pBlock->info.rows = 4;
char data[16] = {0};
char bitmap[4] = {0};
for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
SColumnInfoData colInfo = {0};
colInfo.info.type = TSDB_DATA_TYPE_INT;
colInfo.info.bytes = sizeof(int32_t);
colInfo.info.colId = 1;
colInfo.pData = data;
colInfo.nullbitmap = bitmap;
for (int32_t j = 0; j < pBlock->info.rows; ++j) {
colDataAppendInt32(&colInfo, j, &j);
}
fprintf(stdout, "current working directory:%s\n", path);
strcat(path, "/libudf1.so");
taosArrayPush(pBlock->pDataBlock, &colInfo);
}
UdfcFuncHandle handle;
SEpSet epSet;
setupUdf(udfc, "udf1", &epSet, &handle);
SScalarParam input = {0};
input.numOfRows = pBlock->info.rows;
input.columnData = taosArrayGet(pBlock->pDataBlock, 0);
SScalarParam output = {0};
callUdfScalarFunc(handle, &input, 1, &output);
SSDataBlock block = {0};
SSDataBlock* pBlock = &block;
pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
pBlock->info.numOfCols = 1;
pBlock->info.rows = 4;
char data[16] = {0};
char bitmap[4] = {0};
for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
SColumnInfoData colInfo = {0};
colInfo.info.type = TSDB_DATA_TYPE_INT;
colInfo.info.bytes = sizeof(int32_t);
colInfo.info.colId = 1;
colInfo.pData = data;
colInfo.nullbitmap = bitmap;
for (int32_t j = 0; j < pBlock->info.rows; ++j) {
colDataAppendInt32(&colInfo, j, &j);
}
taosArrayPush(pBlock->pDataBlock, &colInfo);
}
SScalarParam input = {0};
input.numOfRows = pBlock->info.rows;
input.columnData = taosArrayGet(pBlock->pDataBlock, 0);
SScalarParam output = {0};
callUdfScalarFunc(handle, &input, 1 , &output);
SColumnInfoData *col = output.columnData;
for (int32_t i = 0; i < output.numOfRows; ++i) {
fprintf(stderr, "%d\t%d\n" , i, *(int32_t*)(col->pData + i *sizeof(int32_t)));
}
teardownUdf(handle);
udfcClose(udfc);
SColumnInfoData *col = output.columnData;
for (int32_t i = 0; i < output.numOfRows; ++i) {
fprintf(stderr, "%d\t%d\n", i, *(int32_t *)(col->pData + i * sizeof(int32_t)));
}
teardownUdf(handle);
udfcClose();
}

4
source/libs/parser/src/parTranslater.c
View File

@ -3673,7 +3673,9 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pQuery->precision = extractResultTsPrecision((SSelectStmt*)pQuery->pRoot);
if (nodeType(pQuery->pRoot) == QUERY_NODE_SELECT_STMT) {
pQuery->precision = extractResultTsPrecision((SSelectStmt*)pQuery->pRoot);
}
}
if (NULL != pCxt->pDbs) {

16
source/libs/qcom/src/querymsg.c
View File

@ -342,28 +342,20 @@ PROCESS_META_OVER:
int32_t queryProcessQnodeListRsp(void *output, char *msg, int32_t msgSize) {
SQnodeListRsp out = {0};
int32_t code = -1;
int32_t code = 0;
if (NULL == output || NULL == msg || msgSize <= 0) {
code = TSDB_CODE_TSC_INVALID_INPUT;
goto PROCESS_QLIST_OVER;
return code;
}
out.addrsList = (SArray *)output;
if (tDeserializeSQnodeListRsp(msg, msgSize, &out) != 0) {
qError("invalid qnode list rsp msg, msgSize:%d", msgSize);
code = TSDB_CODE_INVALID_MSG;
goto PROCESS_QLIST_OVER;
return code;
}
PROCESS_QLIST_OVER:
if (code != 0) {
tFreeSQnodeListRsp(&out);
out.addrsList = NULL;
}
*(SArray **)output = out.addrsList;
return code;
}

3
source/libs/scalar/inc/sclInt.h
View File

@ -32,6 +32,9 @@ typedef struct SScalarCtx {
#define SCL_DATA_TYPE_DUMMY_HASH 9000
#define SCL_DEFAULT_OP_NUM 10
#define SCL_IS_CONST_NODE(_node) ((NULL == (_node)) || (QUERY_NODE_VALUE == (_node)->type) || (QUERY_NODE_NODE_LIST == (_node)->type))
#define SCL_IS_CONST_CALC(_ctx) (NULL == (_ctx)->pBlockList)
#define sclFatal(...) qFatal(__VA_ARGS__)
#define sclError(...) qError(__VA_ARGS__)
#define sclWarn(...) qWarn(__VA_ARGS__)

158
source/libs/scalar/src/scalar.c
View File

@ -244,23 +244,53 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t
return TSDB_CODE_SUCCESS;
}
int32_t sclInitParamList(SScalarParam **pParams, SNodeList* pParamList, SScalarCtx *ctx, int32_t *rowNum) {
int32_t sclInitParamList(SScalarParam **pParams, SNodeList* pParamList, SScalarCtx *ctx, int32_t *paramNum, int32_t *rowNum) {
int32_t code = 0;
SScalarParam *paramList = taosMemoryCalloc(pParamList->length, sizeof(SScalarParam));
if (NULL == pParamList) {
if (ctx->pBlockList) {
SSDataBlock *pBlock = taosArrayGet(ctx->pBlockList, 0);
*rowNum = pBlock->info.rows;
} else {
*rowNum = 1;
}
*paramNum = 1;
} else {
*paramNum = pParamList->length;
}
SScalarParam *paramList = taosMemoryCalloc(*paramNum, sizeof(SScalarParam));
if (NULL == paramList) {
sclError("calloc %d failed", (int32_t)(pParamList->length * sizeof(SScalarParam)));
sclError("calloc %d failed", (int32_t)((*paramNum) * sizeof(SScalarParam)));
SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
SListCell *cell = pParamList->pHead;
for (int32_t i = 0; i < pParamList->length; ++i) {
if (NULL == cell || NULL == cell->pNode) {
sclError("invalid cell, cell:%p, pNode:%p", cell, cell->pNode);
SCL_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT);
}
if (pParamList) {
SNode *tnode = NULL;
int32_t i = 0;
if (SCL_IS_CONST_CALC(ctx)) {
WHERE_EACH (tnode, pParamList) {
if (!SCL_IS_CONST_NODE(tnode)) {
WHERE_NEXT;
} else {
SCL_ERR_JRET(sclInitParam(tnode, &paramList[i], ctx, rowNum));
ERASE_NODE(pParamList);
}
SCL_ERR_JRET(sclInitParam(cell->pNode, &paramList[i], ctx, rowNum));
cell = cell->pNext;
++i;
}
} else {
FOREACH(tnode, pParamList) {
SCL_ERR_JRET(sclInitParam(tnode, &paramList[i], ctx, rowNum));
++i;
}
}
} else {
paramList[0].numOfRows = *rowNum;
}
if (0 == *rowNum) {
taosMemoryFreeClear(paramList);
}
*pParams = paramList;
@ -299,37 +329,45 @@ _return:
}
int32_t sclExecFunction(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *output) {
if (NULL == node->pParameterList || node->pParameterList->length <= 0) {
sclError("invalid function parameter list, list:%p, paramNum:%d", node->pParameterList, node->pParameterList ? node->pParameterList->length : 0);
SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
}
SScalarFuncExecFuncs ffpSet = {0};
int32_t code = fmGetScalarFuncExecFuncs(node->funcId, &ffpSet);
if (code) {
sclError("fmGetFuncExecFuncs failed, funcId:%d, code:%s", node->funcId, tstrerror(code));
SCL_ERR_RET(code);
}
SScalarParam *params = NULL;
int32_t rowNum = 0;
SCL_ERR_RET(sclInitParamList(&params, node->pParameterList, ctx, &rowNum));
int32_t paramNum = 0;
int32_t code = 0;
SCL_ERR_RET(sclInitParamList(&params, node->pParameterList, ctx, &paramNum, &rowNum));
output->columnData = createColumnInfoData(&node->node.resType, rowNum);
if (output->columnData == NULL) {
sclError("calloc %d failed", (int32_t)(rowNum * output->columnData->info.bytes));
SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
if (fmIsUserDefinedFunc(node->funcId)) {
#if 0
UdfcFuncHandle udfHandle = NULL;
code = (*ffpSet.process)(params, node->pParameterList->length, output);
if (code) {
sclError("scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code));
SCL_ERR_JRET(setupUdf(node->functionName, &udfHandle));
code = callUdfScalarFunc(udfHandle, params, paramNum, output);
teardownUdf(udfHandle);
SCL_ERR_JRET(code);
#endif
} else {
SScalarFuncExecFuncs ffpSet = {0};
code = fmGetScalarFuncExecFuncs(node->funcId, &ffpSet);
if (code) {
sclError("fmGetFuncExecFuncs failed, funcId:%d, code:%s", node->funcId, tstrerror(code));
SCL_ERR_JRET(code);
}
output->columnData = createColumnInfoData(&node->node.resType, rowNum);
if (output->columnData == NULL) {
sclError("calloc %d failed", (int32_t)(rowNum * output->columnData->info.bytes));
SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
code = (*ffpSet.process)(params, paramNum, output);
if (code) {
sclError("scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code));
SCL_ERR_JRET(code);
}
}
_return:
for (int32_t i = 0; i < node->pParameterList->length; ++i) {
for (int32_t i = 0; i < paramNum; ++i) {
// sclFreeParamNoData(params + i);
}
@ -355,8 +393,13 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o
SScalarParam *params = NULL;
int32_t rowNum = 0;
int32_t paramNum = 0;
int32_t code = 0;
SCL_ERR_RET(sclInitParamList(&params, node->pParameterList, ctx, &rowNum));
SCL_ERR_RET(sclInitParamList(&params, node->pParameterList, ctx, &paramNum, &rowNum));
if (NULL == params) {
output->numOfRows = 0;
return TSDB_CODE_SUCCESS;
}
int32_t type = node->node.resType.type;
output->numOfRows = rowNum;
@ -369,25 +412,41 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o
}
bool value = false;
bool complete = true;
for (int32_t i = 0; i < rowNum; ++i) {
for (int32_t m = 0; m < node->pParameterList->length; ++m) {
complete = true;
for (int32_t m = 0; m < paramNum; ++m) {
if (NULL == params[m].columnData) {
complete = false;
continue;
}
char* p = colDataGetData(params[m].columnData, i);
GET_TYPED_DATA(value, bool, params[m].columnData->info.type, p);
if (LOGIC_COND_TYPE_AND == node->condType && (false == value)) {
complete = true;
break;
} else if (LOGIC_COND_TYPE_OR == node->condType && value) {
complete = true;
break;
} else if (LOGIC_COND_TYPE_NOT == node->condType) {
value = !value;
}
}
colDataAppend(output->columnData, i, (char*) &value, false);
if (complete) {
colDataAppend(output->columnData, i, (char*) &value, false);
}
}
if (SCL_IS_CONST_CALC(ctx) && (false == complete)) {
sclFreeParam(output);
output->numOfRows = 0;
}
_return:
for (int32_t i = 0; i < node->pParameterList->length; ++i) {
for (int32_t i = 0; i < paramNum; ++i) {
// sclFreeParamNoData(params + i);
}
@ -426,6 +485,17 @@ _return:
EDealRes sclRewriteFunction(SNode** pNode, SScalarCtx *ctx) {
SFunctionNode *node = (SFunctionNode *)*pNode;
SNode* tnode = NULL;
if (fmIsUserDefinedFunc(node->funcId)) {
return DEAL_RES_CONTINUE;
}
FOREACH(tnode, node->pParameterList) {
if (!SCL_IS_CONST_NODE(tnode)) {
return DEAL_RES_CONTINUE;
}
}
SScalarParam output = {0};
ctx->code = sclExecFunction(node, ctx, &output);
@ -470,6 +540,10 @@ EDealRes sclRewriteLogic(SNode** pNode, SScalarCtx *ctx) {
return DEAL_RES_ERROR;
}
if (0 == output.numOfRows) {
return DEAL_RES_CONTINUE;
}
SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE);
if (NULL == res) {
sclError("make value node failed");
@ -498,6 +572,14 @@ EDealRes sclRewriteLogic(SNode** pNode, SScalarCtx *ctx) {
EDealRes sclRewriteOperator(SNode** pNode, SScalarCtx *ctx) {
SOperatorNode *node = (SOperatorNode *)*pNode;
if (!SCL_IS_CONST_NODE(node->pLeft)) {
return DEAL_RES_CONTINUE;
}
if (!SCL_IS_CONST_NODE(node->pRight)) {
return DEAL_RES_CONTINUE;
}
SScalarParam output = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))};
ctx->code = sclExecOperator(node, ctx, &output);
if (ctx->code) {
@ -530,7 +612,7 @@ EDealRes sclRewriteOperator(SNode** pNode, SScalarCtx *ctx) {
}
EDealRes sclConstantsRewriter(SNode** pNode, void* pContext) {
if (QUERY_NODE_VALUE == nodeType(*pNode) || QUERY_NODE_NODE_LIST == nodeType(*pNode)) {
if (QUERY_NODE_VALUE == nodeType(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode) || QUERY_NODE_NODE_LIST == nodeType(*pNode)) {
return DEAL_RES_CONTINUE;
}

44
source/libs/scalar/src/sclfunc.c
View File

@ -32,8 +32,8 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
float *in = (float *)pInputData->pData;
float *out = (float *)pOutputData->pData;
for (int32_t i = 0; i < pInput->numOfRows; ++i) {
if (colDataIsNull_f(pInputData->nullbitmap, i)) {
colDataSetNull_f(pOutputData->nullbitmap, i);
if (colDataIsNull_s(pInputData, i)) {
colDataAppendNULL(pOutputData, i);
continue;
}
out[i] = (in[i] >= 0)? in[i] : -in[i];
@ -45,8 +45,8 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
double *in = (double *)pInputData->pData;
double *out = (double *)pOutputData->pData;
for (int32_t i = 0; i < pInput->numOfRows; ++i) {
if (colDataIsNull_f(pInputData->nullbitmap, i)) {
colDataSetNull_f(pOutputData->nullbitmap, i);
if (colDataIsNull_s(pInputData, i)) {
colDataAppendNULL(pOutputData, i);
continue;
}
out[i] = (in[i] >= 0)? in[i] : -in[i];
@ -58,8 +58,8 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
int8_t *in = (int8_t *)pInputData->pData;
int8_t *out = (int8_t *)pOutputData->pData;
for (int32_t i = 0; i < pInput->numOfRows; ++i) {
if (colDataIsNull_f(pInputData->nullbitmap, i)) {
colDataSetNull_f(pOutputData->nullbitmap, i);
if (colDataIsNull_s(pInputData, i)) {
colDataAppendNULL(pOutputData, i);
continue;
}
out[i] = (in[i] >= 0)? in[i] : -in[i];
@ -71,8 +71,8 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
int16_t *in = (int16_t *)pInputData->pData;
int16_t *out = (int16_t *)pOutputData->pData;
for (int32_t i = 0; i < pInput->numOfRows; ++i) {
if (colDataIsNull_f(pInputData->nullbitmap, i)) {
colDataSetNull_f(pOutputData->nullbitmap, i);
if (colDataIsNull_s(pInputData, i)) {
colDataAppendNULL(pOutputData, i);
continue;
}
out[i] = (in[i] >= 0)? in[i] : -in[i];
@ -84,8 +84,8 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
int32_t *in = (int32_t *)pInputData->pData;
int32_t *out = (int32_t *)pOutputData->pData;
for (int32_t i = 0; i < pInput->numOfRows; ++i) {
if (colDataIsNull_f(pInputData->nullbitmap, i)) {
colDataSetNull_f(pOutputData->nullbitmap, i);
if (colDataIsNull_s(pInputData, i)) {
colDataAppendNULL(pOutputData, i);
continue;
}
out[i] = (in[i] >= 0)? in[i] : -in[i];
@ -97,8 +97,8 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
int64_t *in = (int64_t *)pInputData->pData;
int64_t *out = (int64_t *)pOutputData->pData;
for (int32_t i = 0; i < pInput->numOfRows; ++i) {
if (colDataIsNull_f(pInputData->nullbitmap, i)) {
colDataSetNull_f(pOutputData->nullbitmap, i);
if (colDataIsNull_s(pInputData, i)) {
colDataAppendNULL(pOutputData, i);
continue;
}
out[i] = (in[i] >= 0)? in[i] : -in[i];
@ -129,8 +129,8 @@ static int32_t doScalarFunctionUnique(SScalarParam *pInput, int32_t inputNum, SS
double *out = (double *)pOutputData->pData;
for (int32_t i = 0; i < pInput->numOfRows; ++i) {
if (colDataIsNull_f(pInputData->nullbitmap, i)) {
colDataSetNull_f(pOutputData->nullbitmap, i);
if (colDataIsNull_s(pInputData, i)) {
colDataAppendNULL(pOutputData, i);
continue;
}
out[i] = valFn(getValueFn(pInputData->pData, i));
@ -157,9 +157,9 @@ static int32_t doScalarFunctionUnique2(SScalarParam *pInput, int32_t inputNum, S
double *out = (double *)pOutputData->pData;
for (int32_t i = 0; i < pInput->numOfRows; ++i) {
if (colDataIsNull_f(pInputData[0]->nullbitmap, i) ||
colDataIsNull_f(pInputData[1]->nullbitmap, 0)) {
colDataSetNull_f(pOutputData->nullbitmap, i);
if (colDataIsNull_s(pInputData[0], i) ||
colDataIsNull_s(pInputData[1], 0)) {
colDataAppendNULL(pOutputData, i);
continue;
}
out[i] = valFn(getValueFn[0](pInputData[0]->pData, i), getValueFn[1](pInputData[1]->pData, 0));
@ -184,8 +184,8 @@ static int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
float *out = (float *)pOutputData->pData;
for (int32_t i = 0; i < pInput->numOfRows; ++i) {
if (colDataIsNull_f(pInputData->nullbitmap, i)) {
colDataSetNull_f(pOutputData->nullbitmap, i);
if (colDataIsNull_s(pInputData, i)) {
colDataAppendNULL(pOutputData, i);
continue;
}
out[i] = f1(in[i]);
@ -198,8 +198,8 @@ static int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
double *out = (double *)pOutputData->pData;
for (int32_t i = 0; i < pInput->numOfRows; ++i) {
if (colDataIsNull_f(pInputData->nullbitmap, i)) {
colDataSetNull_f(pOutputData->nullbitmap, i);
if (colDataIsNull_s(pInputData, i)) {
colDataAppendNULL(pOutputData, i);
continue;
}
out[i] = d1(in[i]);
@ -301,7 +301,7 @@ static int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
for (int32_t i = 0; i < pInput->numOfRows; ++i) {
if (colDataIsNull_s(pInputData, i)) {
colDataSetNull_f(pOutputData->nullbitmap, i);
colDataAppendNULL(pOutputData, i);
continue;
}

116
source/libs/scalar/test/scalar/scalarTests.cpp
View File

@ -137,6 +137,11 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in
rnode->node.resType.bytes = dataBytes;
rnode->dataBlockId = 0;
if (NULL == block) {
*pNode = (SNode *)rnode;
return;
}
if (NULL == *block) {
SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock));
res->info.numOfCols = 3;
@ -889,6 +894,8 @@ TEST(constantTest, int_greater_int_is_true2) {
}
TEST(constantTest, greater_and_lower) {
scltInitLogFile();
SNode *pval1 = NULL, *pval2 = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL, *res = NULL;
bool eRes[5] = {false, false, true, true, true};
int64_t v1 = 333, v2 = 222, v3 = -10, v4 = 20;
@ -913,6 +920,115 @@ TEST(constantTest, greater_and_lower) {
nodesDestroyNode(res);
}
TEST(constantTest, column_and_value1) {
scltInitLogFile();
SNode *pval1 = NULL, *pval2 = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL, *res = NULL;
bool eRes[5] = {false, false, true, true, true};
int64_t v1 = 333, v2 = 222, v3 = -10, v4 = 20;
SNode *list[2] = {0};
scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v1);
scltMakeValueNode(&pval2, TSDB_DATA_TYPE_BIGINT, &v2);
scltMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2);
scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v3);
scltMakeColumnNode(&pval2, NULL, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), 0, NULL);
scltMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2);
list[0] = opNode1;
list[1] = opNode2;
scltMakeLogicNode(&logicNode, LOGIC_COND_TYPE_AND, list, 2);
int32_t code = scalarCalculateConstants(logicNode, &res);
ASSERT_EQ(code, 0);
ASSERT_TRUE(res);
ASSERT_EQ(nodeType(res), QUERY_NODE_LOGIC_CONDITION);
SLogicConditionNode *v = (SLogicConditionNode *)res;
ASSERT_EQ(v->condType, LOGIC_COND_TYPE_AND);
ASSERT_EQ(v->pParameterList->length, 1);
nodesDestroyNode(res);
}
TEST(constantTest, column_and_value2) {
scltInitLogFile();
SNode *pval1 = NULL, *pval2 = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL, *res = NULL;
bool eRes[5] = {false, false, true, true, true};
int64_t v1 = 333, v2 = 222, v3 = -10, v4 = 20;
SNode *list[2] = {0};
scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v1);
scltMakeValueNode(&pval2, TSDB_DATA_TYPE_BIGINT, &v2);
scltMakeOpNode(&opNode1, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2);
scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v3);
scltMakeColumnNode(&pval2, NULL, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), 0, NULL);
scltMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2);
list[0] = opNode1;
list[1] = opNode2;
scltMakeLogicNode(&logicNode, LOGIC_COND_TYPE_AND, list, 2);
int32_t code = scalarCalculateConstants(logicNode, &res);
ASSERT_EQ(code, 0);
ASSERT_TRUE(res);
ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE);
SValueNode *v = (SValueNode *)res;
ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL);
ASSERT_EQ(v->datum.b, false);
nodesDestroyNode(res);
}
TEST(constantTest, column_and_value3) {
scltInitLogFile();
SNode *pval1 = NULL, *pval2 = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL, *res = NULL;
bool eRes[5] = {false, false, true, true, true};
int64_t v1 = 333, v2 = 222, v3 = -10, v4 = 20;
SNode *list[2] = {0};
scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v1);
scltMakeValueNode(&pval2, TSDB_DATA_TYPE_BIGINT, &v2);
scltMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2);
scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v3);
scltMakeColumnNode(&pval2, NULL, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), 0, NULL);
scltMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2);
list[0] = opNode1;
list[1] = opNode2;
scltMakeLogicNode(&logicNode, LOGIC_COND_TYPE_OR, list, 2);
int32_t code = scalarCalculateConstants(logicNode, &res);
ASSERT_EQ(code, 0);
ASSERT_TRUE(res);
ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE);
SValueNode *v = (SValueNode *)res;
ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL);
ASSERT_EQ(v->datum.b, true);
nodesDestroyNode(res);
}
TEST(constantTest, column_and_value4) {
scltInitLogFile();
SNode *pval1 = NULL, *pval2 = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL, *res = NULL;
bool eRes[5] = {false, false, true, true, true};
int64_t v1 = 333, v2 = 222, v3 = -10, v4 = 20;
SNode *list[2] = {0};
scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v1);
scltMakeValueNode(&pval2, TSDB_DATA_TYPE_BIGINT, &v2);
scltMakeOpNode(&opNode1, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2);
scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v3);
scltMakeColumnNode(&pval2, NULL, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), 0, NULL);
scltMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2);
list[0] = opNode1;
list[1] = opNode2;
scltMakeLogicNode(&logicNode, LOGIC_COND_TYPE_OR, list, 2);
int32_t code = scalarCalculateConstants(logicNode, &res);
ASSERT_EQ(code, 0);
ASSERT_TRUE(res);
ASSERT_EQ(nodeType(res), QUERY_NODE_LOGIC_CONDITION);
SLogicConditionNode *v = (SLogicConditionNode *)res;
ASSERT_EQ(v->condType, LOGIC_COND_TYPE_OR);
ASSERT_EQ(v->pParameterList->length, 1);
nodesDestroyNode(res);
}
void makeJsonArrow(SSDataBlock **src, SNode **opNode, void *json, char *key){
char keyVar[32] = {0};
memcpy(varDataVal(keyVar), key, strlen(key));

1
source/libs/sync/src/syncIO.c
View File

@ -170,7 +170,6 @@ static int32_t syncIOStartInternal(SSyncIO *io) {
taosBlockSIGPIPE();
rpcInit();
tsRpcForceTcp = 1;
// cient rpc init
{

6
source/os/src/osSystem.c
View File

@ -119,7 +119,7 @@ int taosSetConsoleEcho(bool on) {
#endif
}
void setTerminalMode() {
void taosSetTerminalMode() {
#if defined(WINDOWS)
#else
@ -152,7 +152,7 @@ void setTerminalMode() {
#endif
}
int32_t getOldTerminalMode() {
int32_t taosGetOldTerminalMode() {
#if defined(WINDOWS)
#else
@ -170,7 +170,7 @@ int32_t getOldTerminalMode() {
#endif
}
void resetTerminalMode() {
void taosResetTerminalMode() {
#if defined(WINDOWS)
#else

1
source/util/src/tarray.c
View File

@ -476,6 +476,7 @@ void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t
return (void*)buf;
}
// todo remove it
// order array<type *>
void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param) {
taosArrayGetSize(pArray) > 8 ? taosArrayQuickSort(pArray, fn, param) : taosArrayInsertSort(pArray, fn, param);

6
source/util/src/tconfig.c
View File

@ -513,6 +513,8 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) {
for (int32_t i = 0; i < size; ++i) {
SConfigItem *pItem = taosArrayGet(pCfg->array, i);
if (tsc && !pItem->tsc) continue;
if (dump && strcmp(pItem->name, "scriptDir") == 0) continue;
if (dump && strcmp(pItem->name, "simDebugFlag") == 0) continue;
tstrncpy(src, cfgStypeStr(pItem->stype), CFG_SRC_PRINT_LEN);
for (int32_t i = 0; i < CFG_SRC_PRINT_LEN; ++i) {
if (src[i] == 0) src[i] = ' ';
@ -551,10 +553,10 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) {
break;
case CFG_DTYPE_FLOAT:
if (dump) {
printf("%s %s %f", src, name, pItem->fval);
printf("%s %s %.2f", src, name, pItem->fval);
printf("\n");
} else {
uInfo("%s %s %f", src, name, pItem->fval);
uInfo("%s %s %.2f", src, name, pItem->fval);
}
break;
case CFG_DTYPE_STRING:

2
source/util/src/ttimer.c
View File

@ -110,7 +110,7 @@ typedef struct time_wheel_t {
tmr_obj_t** slots;
} time_wheel_t;
int32_t tsMaxTmrCtrl = 512;
static int32_t tsMaxTmrCtrl = 512;
static TdThreadOnce tmrModuleInit = PTHREAD_ONCE_INIT;
static TdThreadMutex tmrCtrlMutex;

22
tests/script/tsim/parser/groupby-basic.sim
View File

@ -115,19 +115,19 @@ endi
if $data00 != 10 then
return -1
endi
if $data01 != 0 then
return -1
endi
if $data10 != 10 then
return -1
endi
if $data11 != 1 then
return -1
endi
if $data90 != 10 then
return -1
endi
if $data91 != 9 then
if $data01 != 7 then
return -1
endi
if $data11 != 6 then
return -1
endi
if $data91 != 3 then
return -1
endi
@ -143,16 +143,16 @@ if $row != 10 then
return -1
endi
if $data00 != @22-01-01 00:00:00.000@ then
if $data00 != @22-01-01 00:00:00.007@ then
return -1
endi
if $data01 != 0 then
if $data01 != 7 then
return -1
endi
if $data90 != @22-01-01 00:00:00.009@ then
if $data90 != @22-01-01 00:00:00.003@ then
return -1
endi
if $data91 != 9 then
if $data91 != 3 then
return -1
endi

103
tests/script/tsim/query/crash_sql.sim Normal file
View File

@ -0,0 +1,103 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
$loop_cnt = 0
check_dnode_ready:
$loop_cnt = $loop_cnt + 1
sleep 200
if $loop_cnt == 10 then
print ====> dnode not ready!
return -1
endi
sql show dnodes
print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
if $data00 != 1 then
return -1
endi
if $data04 != ready then
goto check_dnode_ready
endi
sql connect
print =============== create database
sql create database db
sql show databases
if $rows != 2 then
return -1
endi
sql use db
print =============== create super table and child table
sql create table stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int)
sql show stables
print $rows $data00 $data01 $data02
if $rows != 1 then
return -1
endi
sql create table ct1 using stb1 tags ( 1 )
sql create table ct2 using stb1 tags ( 2 )
sql create table ct3 using stb1 tags ( 3 )
sql create table ct4 using stb1 tags ( 4 )
sql show tables
print $rows $data00 $data10 $data20
if $rows != 4 then
return -1
endi
print =============== insert data into child table ct1 (s)
sql insert into ct1 values ( '2022-01-01 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now+1a )
sql insert into ct1 values ( '2022-01-01 01:01:06.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now+2a )
sql insert into ct1 values ( '2022-01-01 01:01:10.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now+3a )
sql insert into ct1 values ( '2022-01-01 01:01:16.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now+4a )
sql insert into ct1 values ( '2022-01-01 01:01:20.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now+5a )
sql insert into ct1 values ( '2022-01-01 01:01:26.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now+6a )
sql insert into ct1 values ( '2022-01-01 01:01:30.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", now+7a )
sql insert into ct1 values ( '2022-01-01 01:01:36.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", now+8a )
print =============== insert data into child table ct2 (d)
sql insert into ct2 values ( '2022-01-01 01:00:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now+1a )
sql insert into ct2 values ( '2022-01-01 10:00:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now+2a )
sql insert into ct2 values ( '2022-01-01 20:00:01.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now+3a )
sql insert into ct2 values ( '2022-01-02 10:00:01.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now+4a )
sql insert into ct2 values ( '2022-01-02 20:00:01.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now+5a )
sql insert into ct2 values ( '2022-01-03 10:00:01.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", now+6a )
sql insert into ct2 values ( '2022-01-03 20:00:01.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", now+7a )
print =============== insert data into child table ct3 (n)
sql insert into ct3 values ( '2021-12-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
sql insert into ct3 values ( '2021-12-31 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now+1a )
sql insert into ct3 values ( '2022-01-01 01:01:06.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now+2a )
sql insert into ct3 values ( '2022-01-07 01:01:10.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now+3a )
sql insert into ct3 values ( '2022-01-31 01:01:16.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now+4a )
sql insert into ct3 values ( '2022-02-01 01:01:20.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now+5a )
sql insert into ct3 values ( '2022-02-28 01:01:26.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now+6a )
sql insert into ct3 values ( '2022-03-01 01:01:30.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" )
sql insert into ct3 values ( '2022-03-08 01:01:36.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" )
print =============== insert data into child table ct4 (y)
sql insert into ct4 values ( '2019-01-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
sql insert into ct4 values ( '2019-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now+1a )
sql insert into ct4 values ( '2019-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now+2a )
sql insert into ct4 values ( '2020-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now+3a )
sql insert into ct4 values ( '2020-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now+4a )
sql insert into ct4 values ( '2020-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now+5a )
sql insert into ct4 values ( '2020-12-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
sql insert into ct4 values ( '2021-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now+6a )
sql insert into ct4 values ( '2021-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" )
sql insert into ct4 values ( '2021-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" )
sql insert into ct4 values ( '2022-02-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" )
sql insert into ct4 values ( '2022-05-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
print ================ start query ======================
print ================ SQL used to cause taosd or taos shell crash
sql select sum(c1) ,count(c1) from ct4 group by c1 having sum(c10) between 0 and 1 ;
#system sh/exec.sh -n dnode1 -s stop -x SIGINT

1
tests/script/tsim/stable/disk.sim
View File

@ -53,6 +53,7 @@ endi
sql select count(tbcol) from $mt
print select count(tbcol) from $mt ===> $data00
if $data00 != $totalNum then
print expect $totalNum , actual: $data00
return -1
endi

161
tests/system-test/2-query/cast.py
View File

@ -6,6 +6,7 @@ import inspect
from util.log import *
from util.sql import *
from util.cases import *
from util.dnodes import *
@ -15,56 +16,50 @@ class TDTestCase:
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor())
def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring
tdSql.prepare()
def __cast_to_bigint(self, col_name, tbname):
__sql = f"select cast({col_name} as bigint), {col_name} from {tbname}"
tdSql.query(sql=__sql)
data_tb_col = [result[1] for result in tdSql.queryResult]
for i in range(len(tdSql.queryRows)):
tdSql.checkData( i, 0, None ) if data_tb_col[i] is None else tdSql.checkData( i, 0, int(data_tb_col[i]) )
tdLog.printNoPrefix("==========step1:create table")
tdSql.execute(
'''create table stb1
(ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
tags (t1 int)
'''
)
tdSql.execute(
'''
create table t1
(ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
'''
)
for i in range(4):
tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
def __range_to_bigint(self,cols,tables):
for col in cols:
for table in tables:
self.__cast_to_bigint(col_name=col, tbname=table)
tdLog.printNoPrefix("==========step2:insert data")
for i in range(9):
tdSql.execute(
f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
)
tdSql.execute(
f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
)
tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )")
tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )")
def __cast_to_timestamp(self, col_name, tbname):
__sql = f"select cast({col_name} as timestamp), {col_name} from {tbname}"
tdSql.query(sql=__sql)
data_tb_col = [result[1] for result in tdSql.queryResult]
for i in range(len(tdSql.queryRows)):
if data_tb_col[i] is None:
tdSql.checkData( i, 0 , None )
if (col_name == "c2" or col_name == "double" ) and tbname == "t1" and i == 10:
continue
else:
utc_zone = datetime.timezone.utc
utc_8 = datetime.timezone(datetime.timedelta(hours=8))
date_init_stamp = datetime.datetime.utcfromtimestamp(data_tb_col[i]/1000)
date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f")
tdSql.checkData( i, 0, date_data)
tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
def __range_to_timestamp(self, cols, tables):
for col in cols:
for table in tables:
self.__cast_to_timestamp(col_name=col, tbname=table)
tdSql.execute(
f'''insert into t1 values
( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a )
( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a )
( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a )
( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a )
( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a )
( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a )
( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" )
( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" )
( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" )
( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
'''
)
def __test_bigint(self):
__table_list = ["ct1", "ct4", "t1"]
__col_list = ["c1","c2","c3","c4","c5","c6","c7","c10","c1+c2"]
self.__range_to_bigint(cols=__col_list, tables=__table_list)
def __test_timestamp(self):
__table_list = ["ct1", "ct4", "t1"]
__col_list = ["c1","c2","c3","c4","c5","c6","c7","c1+c2"]
self.__range_to_timestamp(cols=__col_list, tables=__table_list)
def all_test(self):
tdSql.query("select c1 from ct4")
data_ct4_c1 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)]
@ -82,9 +77,9 @@ class TDTestCase:
tdLog.printNoPrefix("==========step5: cast int to binary, expect changes to str(int) ")
tdSql.query("select cast(c1 as binary(32)) as b from ct4")
for i in range(len(data_ct4_c1)):
tdSql.checkData( i, 0, str(data_ct4_c1[i]) )
#tdSql.query("select cast(c1 as binary(32)) as b from ct4")
#for i in range(len(data_ct4_c1)):
# tdSql.checkData( i, 0, str(data_ct4_c1[i]) )
tdSql.query("select cast(c1 as binary(32)) as b from t1")
for i in range(len(data_t1_c1)):
tdSql.checkData( i, 0, str(data_t1_c1[i]) )
@ -240,7 +235,7 @@ class TDTestCase:
tdSql.checkData( i, 0, date_data)
tdLog.printNoPrefix("==========step16: cast smallint to bigint, expect no changes")
tdLog.printNoPrefix("==========step16: cast tinyint to bigint, expect no changes")
tdSql.query("select c4 from ct4")
data_ct4_c4 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)]
tdSql.query("select c4 from t1")
@ -254,7 +249,7 @@ class TDTestCase:
tdSql.checkData( i, 0, data_t1_c4[i])
tdLog.printNoPrefix("==========step17: cast smallint to binary, expect changes to str(int) ")
tdLog.printNoPrefix("==========step17: cast tinyint to binary, expect changes to str(int) ")
tdSql.query("select cast(c4 as binary(32)) as b from ct4")
for i in range(len(data_ct4_c4)):
@ -263,7 +258,7 @@ class TDTestCase:
for i in range(len(data_t1_c4)):
tdSql.checkData( i, 0, str(data_t1_c4[i]) )
tdLog.printNoPrefix("==========step18: cast smallint to nchar, expect changes to str(int) ")
tdLog.printNoPrefix("==========step18: cast tinyint to nchar, expect changes to str(int) ")
tdSql.query("select cast(c4 as nchar(32)) as b from ct4")
for i in range(len(data_ct4_c4)):
@ -272,7 +267,7 @@ class TDTestCase:
for i in range(len(data_t1_c4)):
tdSql.checkData( i, 0, str(data_t1_c4[i]) )
tdLog.printNoPrefix("==========step19: cast smallint to timestamp, expect changes to timestamp ")
tdLog.printNoPrefix("==========step19: cast tinyint to timestamp, expect changes to timestamp ")
tdSql.query("select cast(c4 as timestamp) as b from ct4")
for i in range(len(data_ct4_c4)):
@ -624,7 +619,67 @@ class TDTestCase:
tdSql.error("select cast(c8 as timestamp ) as b from ct4")
tdSql.error("select cast(c9 as timestamp ) as b from ct4")
tdSql.error("select cast(c9 as binary(64) ) as b from ct4")
pass
def run(self):
tdSql.prepare()
tdLog.printNoPrefix("==========step1:create table")
tdSql.execute(
'''create table stb1
(ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
tags (t1 int)
'''
)
tdSql.execute(
'''
create table t1
(ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
'''
)
for i in range(4):
tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
tdLog.printNoPrefix("==========step2:insert data")
for i in range(9):
tdSql.execute(
f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
)
tdSql.execute(
f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
)
tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )")
tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )")
tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute(
f'''insert into t1 values
( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a )
( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a )
( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a )
( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a )
( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a )
( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a )
( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" )
( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" )
( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" )
( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
'''
)
self.all_test()
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.execute("use db")
self.all_test()
def stop(self):
tdSql.close()

232
tests/system-test/2-query/sum.py Normal file
View File

@ -0,0 +1,232 @@
from util.log import *
from util.sql import *
from util.cases import *
from util.dnodes import *
INT_COL = "c1"
BINT_COL = "c2"
SINT_COL = "c3"
TINT_COL = "c4"
FLOAT_COL = "c5"
DOUBLE_COL = "c6"
BOOL_COL = "c7"
BINARY_COL = "c8"
NCHAR_COL = "c9"
TS_COL = "c10"
NUM_COL = [INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ]
UN_NUM_COL = [BOOL_COL, BINARY_COL, NCHAR_COL, ]
TS_TYPE_COL = [TS_COL]
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor())
def __sum_condition(self):
sum_condition = []
for num_col in NUM_COL:
sum_condition.extend(
(
num_col,
f"ceil( {num_col} )",
)
)
sum_condition.extend( f"{num_col} + {num_col_2}" for num_col_2 in NUM_COL )
sum_condition.extend( f"{num_col} + {un_num_col} " for un_num_col in UN_NUM_COL )
sum_condition.append(1)
return sum_condition
def __where_condition(self, col):
return f" where abs( {col} ) < 1000000 "
def __group_condition(self, col, having = ""):
return f" group by {col} having {having}" if having else f" group by {col} "
def __sum_current_check(self, tbname):
sum_condition = self.__sum_condition()
for condition in sum_condition:
where_condition = self.__where_condition(condition)
group_condition = self.__group_condition(condition, having=f"{condition} is not null " )
tdSql.query(f"select {condition} from {tbname} {where_condition} ")
datas = [tdSql.getData(i,0) for i in range(tdSql.queryRows)]
sum_data = sum(filter(None, datas))
tdSql.query(f"select sum( {condition} ) from {tbname} {where_condition} ")
tdSql.checkData(0, 0, sum_data)
tdSql.query(f"select {condition} from {tbname} {where_condition} {group_condition} ")
def __sum_err_check(self,tbanme):
sqls = []
for un_num_col in UN_NUM_COL:
sqls.extend(
(
f"select sum( {un_num_col} ) from {tbanme} ",
f"select sum(ceil( {un_num_col} )) from {tbanme} ",
)
)
sqls.extend( f"select sum( {un_num_col} + {un_num_col_2} ) from {tbanme} " for un_num_col_2 in UN_NUM_COL )
sqls.extend( f"select sum( {num_col} + {ts_col} ) from {tbanme} " for num_col in NUM_COL for ts_col in TS_TYPE_COL)
sqls.extend(
(
f"select sum() from {tbanme} ",
f"select sum(*) from {tbanme} ",
f"select sum(ccccccc) from {tbanme} ",
f"select sum('test') from {tbanme} ",
)
)
return sqls
def __test_current(self):
tdLog.printNoPrefix("==========current sql condition check , must return query ok==========")
tbname = ["ct1", "ct2", "ct4", "t1"]
for tb in tbname:
self.__sum_current_check(tb)
tdLog.printNoPrefix(f"==========current sql condition check in {tb} over==========")
def __test_error(self):
tdLog.printNoPrefix("==========err sql condition check , must return error==========")
tbname = ["ct1", "ct2", "ct4", "t1"]
for tb in tbname:
for errsql in self.__sum_err_check(tb):
tdSql.error(sql=errsql)
tdLog.printNoPrefix(f"==========err sql condition check in {tb} over==========")
def all_test(self):
self.__test_current()
self.__test_error()
def __create_tb(self):
tdSql.prepare()
tdLog.printNoPrefix("==========step1:create table")
create_stb_sql = f'''create table stb1(
ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint,
{FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool,
{BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp
) tags (t1 int)
'''
create_ntb_sql = f'''create table t1(
ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint,
{FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool,
{BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp
)
'''
tdSql.execute(create_stb_sql)
tdSql.execute(create_ntb_sql)
for i in range(4):
tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
def __insert_data(self, rows):
for i in range(9):
tdSql.execute(
f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
)
tdSql.execute(
f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
)
tdSql.execute(
f"insert into ct2 values ( now()-{i*90}d, {-1*i}, {-11111*i}, {-111*i}, {-11*i}, {-1.11*i}, {-11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
)
tdSql.execute(
'''insert into ct1 values
( now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )
( now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )
'''
)
tdSql.execute(
f'''insert into ct4 values
( now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
( now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
( now()+{rows * 9}d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
(
now()+{rows * 9-10}d, {pow(2,31)-pow(2,15)}, {pow(2,63)-pow(2,30)}, 32767, 127,
{ 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nachar_limit-1", now()-1d
)
(
now()+{rows * 9-20}d, {pow(2,31)-pow(2,16)}, {pow(2,63)-pow(2,31)}, 32766, 126,
{ 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nachar_limit-2", now()-2d
)
'''
)
tdSql.execute(
f'''insert into ct2 values
( now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
( now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
( now()+{rows * 9}d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
(
now()+{rows * 9-10}d, { -1 * pow(2,31) + pow(2,15) }, { -1 * pow(2,63) + pow(2,30) }, -32766, -126,
{ -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nachar_limit-1", now()-1d
)
(
now()+{rows * 9-20}d, { -1 * pow(2,31) + pow(2,16) }, { -1 * pow(2,63) + pow(2,31) }, -32767, -127,
{ - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nachar_limit-2", now()-2d
)
'''
)
for i in range(rows):
insert_data = f'''insert into t1 values
( now()-{i}h, {i}, {i}, { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2},
"binary_{i}", "nchar_{i}", now()-{i}s )
'''
tdSql.execute(insert_data)
tdSql.execute(
f'''insert into t1 values
( now() + 3h, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
( now()-{ ( rows // 2 ) * 60 + 30 }m, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
( now()-{rows}h, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
( now() + 2h, { pow(2,31) - pow(2,15) }, { pow(2,63) - pow(2,30) }, 32767, 127,
{ 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 },
"binary_limit-1", "nachar_limit-1", now()-1d
)
(
now() + 1h , { pow(2,31) - pow(2,16) }, { pow(2,63) - pow(2,31) }, 32766, 126,
{ 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 },
"binary_limit-2", "nachar_limit-2", now()-2d
)
'''
)
def run(self):
tdSql.prepare()
tdLog.printNoPrefix("==========step1:create table")
self.__create_tb()
tdLog.printNoPrefix("==========step2:insert data")
self.__insert_data(100)
tdLog.printNoPrefix("==========step3:all check")
self.all_test()
# tdDnodes.stop(1)
# tdDnodes.start(1)
# tdSql.execute("use db")
# tdLog.printNoPrefix("==========step4:after wal, all check again ")
# self.all_test()
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

4
tests/system-test/fulltest.sh
View File

@ -2,6 +2,6 @@
set -e
#python3 ./test.py -f 2-query/between.py
python3 ./test.py -f 2-query/distinct.py
#python3 ./test.py -f 2-query/distinct.py
python3 ./test.py -f 2-query/varchar.py
python3 ./test.py -f 2-query/cast.py
#python3 ./test.py -f 2-query/cast.py

93
tools/shell/inc/shell.h
View File

@ -1,93 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_SHELL_H_
#define _TD_SHELL_H_
#include "os.h"
#include "taos.h"
#include "taosdef.h"
#define MAX_USERNAME_SIZE 64
#define MAX_DBNAME_SIZE 64
#define MAX_IP_SIZE 20
#define MAX_HISTORY_SIZE 1000
#define MAX_COMMAND_SIZE 1048586
#define HISTORY_FILE ".taos_history"
#define DEFAULT_RES_SHOW_NUM 100
typedef struct SShellHistory {
char* hist[MAX_HISTORY_SIZE];
int hstart;
int hend;
} SShellHistory;
typedef struct SShellArguments {
char* host;
char* password;
char* user;
char* auth;
char* database;
char* timezone;
bool is_raw_time;
bool is_use_passwd;
bool dump_config;
char file[TSDB_FILENAME_LEN];
char dir[TSDB_FILENAME_LEN];
int threadNum;
int check;
bool status;
bool verbose;
char* commands;
int abort;
int port;
int pktLen;
int pktNum;
char* pktType;
char* netTestRole;
} SShellArguments;
/**************** Function declarations ****************/
extern void shellParseArgument(int argc, char* argv[], SShellArguments* arguments);
extern TAOS* shellInit(SShellArguments* args);
extern void* shellLoopQuery(void* arg);
extern void taos_error(TAOS_RES* tres, int64_t st);
extern int regex_match(const char* s, const char* reg, int cflags);
int32_t shellReadCommand(TAOS* con, char command[]);
int32_t shellRunCommand(TAOS* con, char* command);
void shellRunCommandOnServer(TAOS* con, char command[]);
void read_history();
void write_history();
void source_file(TAOS* con, char* fptr);
void source_dir(TAOS* con, SShellArguments* args);
void get_history_path(char* history);
void shellCheck(TAOS* con, SShellArguments* args);
void cleanup_handler(void* arg);
void exitShell();
int shellDumpResult(TAOS_RES* con, char* fname, int* error_no, bool printMode);
void shellGetGrantInfo(void *con);
int isCommentLine(char *line);
/**************** Global variable declarations ****************/
extern char PROMPT_HEADER[];
extern char CONTINUE_PROMPT[];
extern int prompt_size;
extern SShellHistory history;
extern SShellArguments args;
extern int64_t result;
#endif

55
tools/shell/inc/shellCommand.h
View File

@ -1,55 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_SHELL_COMMAND_H_
#define _TD_SHELL_COMMAND_H_
#include "shell.h"
#define LEFT 1
#define RIGHT 2
#define UP 3
#define DOWN 4
typedef struct Command Command;
struct Command {
char * buffer;
char * command;
unsigned commandSize;
unsigned bufferSize;
unsigned cursorOffset;
unsigned screenOffset;
unsigned endOffset;
};
extern void backspaceChar(Command *cmd);
extern void clearLineBefore(Command *cmd);
extern void clearLineAfter(Command *cmd);
extern void deleteChar(Command *cmd);
extern void moveCursorLeft(Command *cmd);
extern void moveCursorRight(Command *cmd);
extern void positionCursorHome(Command *cmd);
extern void positionCursorEnd(Command *cmd);
extern void showOnScreen(Command *cmd);
extern void updateBuffer(Command *cmd);
extern int isReadyGo(Command *cmd);
extern void resetCommand(Command *cmd, const char s[]);
int countPrefixOnes(unsigned char c);
void clearScreen(int ecmd_pos, int cursor_pos);
void printChar(char c, int times);
void positionCursor(int step, int direction);
#endif

117
tools/shell/inc/shellInt.h Normal file
View File

@ -0,0 +1,117 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_SHELL_INT_H_
#define _TD_SHELL_INT_H_
#include "os.h"
#include "taos.h"
#include "taosdef.h"
#include "taoserror.h"
#include "tconfig.h"
#include "tglobal.h"
#include "trpc.h"
#include "ttypes.h"
#include "tutil.h"
#define SHELL_MAX_HISTORY_SIZE 1000
#define SHELL_MAX_COMMAND_SIZE 1048586
#define SHELL_HISTORY_FILE ".taos_history"
#define SHELL_DEFAULT_RES_SHOW_NUM 100
#define SHELL_DEFAULT_MAX_BINARY_DISPLAY_WIDTH 30
#define SHELL_MAX_PKG_LEN 2 * 1024 * 1024
#define SHELL_MIN_PKG_LEN 1
#define SHELL_DEF_PKG_LEN 1024
#define SHELL_MAX_PKG_NUM 1 * 1024 * 1024
#define SHELL_MIN_PKG_NUM 1
#define SHELL_DEF_PKG_NUM 100
typedef struct {
char* hist[SHELL_MAX_HISTORY_SIZE];
char file[TSDB_FILENAME_LEN];
int32_t hstart;
int32_t hend;
} SShellHistory;
typedef struct {
const char* host;
const char* user;
const char* auth;
const char* database;
const char* cfgdir;
const char* commands;
const char* netrole;
char file[PATH_MAX];
char password[TSDB_USET_PASSWORD_LEN];
bool is_gen_auth;
bool is_raw_time;
bool is_version;
bool is_dump_config;
bool is_check;
bool is_startup;
bool is_help;
uint16_t port;
int32_t pktLen;
int32_t pktNum;
int32_t displayWidth;
int32_t abort;
} SShellArgs;
typedef struct {
const char* clientVersion;
const char* promptHeader;
const char* promptContinue;
const char* osname;
int32_t promptSize;
char programVersion[32];
} SShellOsDetails;
typedef struct {
SShellArgs args;
SShellHistory history;
SShellOsDetails info;
TAOS* conn;
TdThread pid;
tsem_t cancelSem;
int64_t result;
} SShellObj;
// shellArguments.c
int32_t shellParseArgs(int32_t argc, char* argv[]);
// shellCommand.c
int32_t shellReadCommand(char* command);
// shellEngine.c
int32_t shellExecute();
// shellUtil.c
int32_t shellCheckIntSize();
void shellPrintVersion();
void shellPrintHelp();
void shellGenerateAuth();
void shellDumpConfig();
void shellCheckServerStatus();
bool shellRegexMatch(const char* s, const char* reg, int32_t cflags);
void shellExit();
// shellNettest.c
void shellTestNetWork();
// shellMain.c
extern SShellObj shell;
extern void taos_init();
#endif /*_TD_SHELL_INT_H_*/

141
tools/shell/inc/syncMsg.h
View File

@ -1,141 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_SYNC_MSG_H
#define TDENGINE_SYNC_MSG_H
#ifdef __cplusplus
extern "C" {
#endif
#include "tsync.h"
typedef enum {
TAOS_SMSG_START = 0,
TAOS_SMSG_SYNC_DATA = 1,
TAOS_SMSG_SYNC_DATA_RSP = 2,
TAOS_SMSG_SYNC_FWD = 3,
TAOS_SMSG_SYNC_FWD_RSP = 4,
TAOS_SMSG_SYNC_REQ = 5,
TAOS_SMSG_SYNC_REQ_RSP = 6,
TAOS_SMSG_SYNC_MUST = 7,
TAOS_SMSG_SYNC_MUST_RSP = 8,
TAOS_SMSG_STATUS = 9,
TAOS_SMSG_STATUS_RSP = 10,
TAOS_SMSG_SETUP = 11,
TAOS_SMSG_SETUP_RSP = 12,
TAOS_SMSG_SYNC_FILE = 13,
TAOS_SMSG_SYNC_FILE_RSP = 14,
TAOS_SMSG_TEST = 15,
TAOS_SMSG_END = 16
} ESyncMsgType;
typedef enum {
SYNC_STATUS_BROADCAST,
SYNC_STATUS_BROADCAST_RSP,
SYNC_STATUS_SETUP_CONN,
SYNC_STATUS_SETUP_CONN_RSP,
SYNC_STATUS_EXCHANGE_DATA,
SYNC_STATUS_EXCHANGE_DATA_RSP,
SYNC_STATUS_CHECK_ROLE,
SYNC_STATUS_CHECK_ROLE_RSP
} ESyncStatusType;
#pragma pack(push, 1)
typedef struct {
int8_t type; // msg type
int8_t protocol; // protocol version
uint16_t signature; // fixed value
int32_t code; //
int32_t cId; // cluster Id
int32_t vgId; // vg ID
int32_t len; // content length, does not include head
uint32_t cksum;
} SSyncHead;
typedef struct {
SSyncHead head;
uint16_t port;
uint16_t tranId;
int32_t sourceId; // only for arbitrator
char fqdn[TSDB_FQDN_LEN];
} SSyncMsg;
typedef struct {
SSyncHead head;
int8_t sync;
int8_t reserved;
uint16_t tranId;
int8_t reserverd[4];
} SSyncRsp;
typedef struct {
int8_t role;
uint64_t version;
} SPeerStatus;
typedef struct {
SSyncHead head;
int8_t role;
int8_t ack;
int8_t type;
int8_t reserved[3];
uint16_t tranId;
uint64_t version;
SPeerStatus peersStatus[TAOS_SYNC_MAX_REPLICA];
} SPeersStatus;
typedef struct {
SSyncHead head;
uint64_t fversion;
} SFileVersion;
typedef struct {
SSyncHead head;
int8_t ack;
} SFileAck;
typedef struct {
SSyncHead head;
uint64_t version;
int32_t code;
} SFwdRsp;
#pragma pack(pop)
#define SYNC_PROTOCOL_VERSION 1
#define SYNC_SIGNATURE ((uint16_t)(0xCDEF))
extern char *statusType[];
uint16_t syncGenTranId();
int32_t syncCheckHead(SSyncHead *pHead);
void syncBuildSyncFwdMsg(SSyncHead *pHead, int32_t vgId, int32_t len);
void syncBuildSyncFwdRsp(SFwdRsp *pMsg, int32_t vgId, uint64_t version, int32_t code);
void syncBuildSyncReqMsg(SSyncMsg *pMsg, int32_t vgId);
void syncBuildSyncDataMsg(SSyncMsg *pMsg, int32_t vgId);
void syncBuildSyncSetupMsg(SSyncMsg *pMsg, int32_t vgId);
void syncBuildPeersStatus(SPeersStatus *pMsg, int32_t vgId);
void syncBuildSyncTestMsg(SSyncMsg *pMsg, int32_t vgId);
void syncBuildFileAck(SFileAck *pMsg, int32_t vgId);
void syncBuildFileVersion(SFileVersion *pMsg, int32_t vgId);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_VNODEPEER_H

127
tools/shell/inc/tsync.h
View File

@ -1,127 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_SYNC_H
#define TDENGINE_SYNC_H
#ifdef __cplusplus
extern "C" {
#endif
#define TAOS_SYNC_MAX_REPLICA 5
#define TAOS_SYNC_MAX_INDEX 0x7FFFFFFF
typedef enum {
TAOS_SYNC_ROLE_OFFLINE = 0,
TAOS_SYNC_ROLE_UNSYNCED = 1,
TAOS_SYNC_ROLE_SYNCING = 2,
TAOS_SYNC_ROLE_SLAVE = 3,
TAOS_SYNC_ROLE_MASTER = 4
} ESyncRole;
typedef enum {
TAOS_SYNC_STATUS_INIT = 0,
TAOS_SYNC_STATUS_START = 1,
TAOS_SYNC_STATUS_FILE = 2,
TAOS_SYNC_STATUS_CACHE = 3
} ESyncStatus;
typedef struct {
uint32_t nodeId; // node ID assigned by TDengine
uint16_t nodePort; // node sync Port
char nodeFqdn[TSDB_FQDN_LEN]; // node FQDN
} SNodeInfo;
typedef struct {
int8_t quorum; // number of confirms required, >=1
int8_t replica; // number of replications, >=1
SNodeInfo nodeInfo[TAOS_SYNC_MAX_REPLICA];
} SSyncCfg;
typedef struct {
int32_t selfIndex;
uint32_t nodeId[TAOS_SYNC_MAX_REPLICA];
int32_t role[TAOS_SYNC_MAX_REPLICA];
} SNodesRole;
// get the wal file from index or after
// return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file
typedef int32_t (*FGetWalInfo)(int32_t vgId, char *fileName, int64_t *fileId);
// when a forward pkt is received, call this to handle data
typedef int32_t (*FWriteToCache)(int32_t vgId, void *pHead, int32_t qtype, void *pMsg);
// when forward is confirmed by peer, master call this API to notify app
typedef void (*FConfirmForward)(int32_t vgId, void *mhandle, int32_t code);
// when role is changed, call this to notify app
typedef void (*FNotifyRole)(int32_t vgId, int8_t role);
// if a number of retrieving data failed, call this to start flow control
typedef void (*FNotifyFlowCtrl)(int32_t vgId, int32_t level);
// when data file is synced successfully, notity app
typedef void (*FStartSyncFile)(int32_t vgId);
typedef void (*FStopSyncFile)(int32_t vgId, uint64_t fversion);
// get file version
typedef int32_t (*FGetVersion)(int32_t vgId, uint64_t *fver, uint64_t *vver);
typedef int32_t (*FSendFile)(void *tsdb, SOCKET socketFd);
typedef int32_t (*FRecvFile)(void *tsdb, SOCKET socketFd);
typedef struct {
int32_t vgId; // vgroup ID
uint64_t version; // initial version
SSyncCfg syncCfg; // configuration from mgmt
char path[TSDB_FILENAME_LEN]; // path to the file
void * pTsdb;
FGetWalInfo getWalInfoFp;
FWriteToCache writeToCacheFp;
FConfirmForward confirmForward;
FNotifyRole notifyRoleFp;
FNotifyFlowCtrl notifyFlowCtrlFp;
FStartSyncFile startSyncFileFp;
FStopSyncFile stopSyncFileFp;
FGetVersion getVersionFp;
FSendFile sendFileFp;
FRecvFile recvFileFp;
} SSyncInfo;
typedef void *tsync_h;
int32_t syncInit();
void syncCleanUp();
int64_t syncStart(const SSyncInfo *);
void syncStop(int64_t rid);
int32_t syncReconfig(int64_t rid, const SSyncCfg *);
int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int32_t qtype, bool force);
void syncConfirmForward(int64_t rid, uint64_t version, int32_t code, bool force);
void syncRecover(int64_t rid); // recover from other nodes:
int32_t syncGetNodesRole(int64_t rid, SNodesRole *);
extern char *syncRole[];
//global configurable parameters
extern int32_t sDebugFlag;
extern char tsArbitrator[];
extern uint16_t tsSyncPort;
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_SYNC_H

202
tools/shell/src/backup/shellCheck.c
View File

@ -1,202 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _GNU_SOURCE
#define _XOPEN_SOURCE
#define _DEFAULT_SOURCE
#include "os.h"
#include "shell.h"
#include "shellCommand.h"
#include "tglobal.h"
#include "tutil.h"
#define SHELL_SQL_LEN 1024
static int32_t tbNum = 0;
static int32_t tbMallocNum = 0;
static char ** tbNames = NULL;
static int32_t checkedNum = 0;
static int32_t errorNum = 0;
typedef struct {
TdThread threadID;
int threadIndex;
int totalThreads;
void * taos;
char * db;
} ShellThreadObj;
static int32_t shellUseDb(TAOS *con, char *db) {
if (db == NULL) {
fprintf(stdout, "no dbname input\n");
return -1;
}
char sql[SHELL_SQL_LEN] = {0};
snprintf(sql, SHELL_SQL_LEN, "use %s", db);
TAOS_RES *pSql = taos_query(con, sql);
int32_t code = taos_errno(pSql);
if (code != 0) {
fprintf(stdout, "failed to execute sql:%s since %s", sql, taos_errstr(pSql));
}
taos_free_result(pSql);
return code;
}
static int32_t shellShowTables(TAOS *con, char *db) {
char sql[SHELL_SQL_LEN] = {0};
snprintf(sql, SHELL_SQL_LEN, "show %s.tables", db);
TAOS_RES *pSql = taos_query(con, sql);
int32_t code = taos_errno(pSql);
if (code != 0) {
fprintf(stdout, "failed to execute sql:%s since %s\n", sql, taos_errstr(pSql));
} else {
TAOS_ROW row;
while ((row = taos_fetch_row(pSql))) {
int32_t tbIndex = tbNum++;
if (tbMallocNum < tbNum) {
tbMallocNum = (tbMallocNum * 2 + 1);
char** tbNames1 = taosMemoryRealloc(tbNames, tbMallocNum * sizeof(char *));
if (tbNames1 == NULL) {
fprintf(stdout, "failed to malloc tablenames, num:%d\n", tbMallocNum);
code = TSDB_CODE_TSC_OUT_OF_MEMORY;
break;
}
tbNames = tbNames1;
}
tbNames[tbIndex] = taosMemoryMalloc(TSDB_TABLE_NAME_LEN);
strncpy(tbNames[tbIndex], (const char *)row[0], TSDB_TABLE_NAME_LEN);
if (tbIndex % 100000 == 0 && tbIndex != 0) {
fprintf(stdout, "%d tablenames fetched\n", tbIndex);
}
}
}
taos_free_result(pSql);
fprintf(stdout, "total %d tablenames fetched, over\n", tbNum);
return code;
}
static void shellFreeTbnames() {
for (int32_t i = 0; i < tbNum; ++i) {
taosMemoryFree(tbNames[i]);
}
taosMemoryFree(tbNames);
}
static void *shellCheckThreadFp(void *arg) {
ShellThreadObj *pThread = (ShellThreadObj *)arg;
setThreadName("shellCheckThrd");
int32_t interval = tbNum / pThread->totalThreads + 1;
int32_t start = pThread->threadIndex * interval;
int32_t end = (pThread->threadIndex + 1) * interval;
if (end > tbNum) end = tbNum + 1;
char file[32] = {0};
snprintf(file, 32, "tb%d.txt", pThread->threadIndex);
TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (!fp) {
fprintf(stdout, "failed to open %s, reason:%s", file, strerror(errno));
return NULL;
}
char sql[SHELL_SQL_LEN];
for (int32_t t = start; t < end; ++t) {
char *tbname = tbNames[t];
if (tbname == NULL) break;
snprintf(sql, SHELL_SQL_LEN, "select * from %s limit 1", tbname);
TAOS_RES *pSql = taos_query(pThread->taos, sql);
int32_t code = taos_errno(pSql);
if (code != 0) {
int32_t len = snprintf(sql, SHELL_SQL_LEN, "drop table %s.%s;\n", pThread->db, tbname);
taosWriteFile(pFile, sql, len);
atomic_add_fetch_32(&errorNum, 1);
}
int32_t cnum = atomic_add_fetch_32(&checkedNum, 1);
if (cnum % 5000 == 0 && cnum != 0) {
fprintf(stdout, "%d tables checked\n", cnum);
}
taos_free_result(pSql);
}
taosFsync(pFile);
taosCloseFile(&pFile);
return NULL;
}
static void shellRunCheckThreads(TAOS *con, SShellArguments *_args) {
TdThreadAttr thattr;
ShellThreadObj *threadObj = (ShellThreadObj *)taosMemoryCalloc(_args->threadNum, sizeof(ShellThreadObj));
for (int t = 0; t < _args->threadNum; ++t) {
ShellThreadObj *pThread = threadObj + t;
pThread->threadIndex = t;
pThread->totalThreads = _args->threadNum;
pThread->taos = con;
pThread->db = _args->database;
taosThreadAttrInit(&thattr);
taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&(pThread->threadID), &thattr, shellCheckThreadFp, (void *)pThread) != 0) {
fprintf(stderr, "ERROR: thread:%d failed to start\n", pThread->threadIndex);
exit(0);
}
}
for (int t = 0; t < _args->threadNum; ++t) {
taosThreadJoin(threadObj[t].threadID, NULL);
}
for (int t = 0; t < _args->threadNum; ++t) {
taos_close(threadObj[t].taos);
}
taosMemoryFree(threadObj);
}
void shellCheck(TAOS *con, SShellArguments *_args) {
int64_t start = taosGetTimestampMs();
if (shellUseDb(con, _args->database) != 0) {
shellFreeTbnames();
return;
}
if (shellShowTables(con, _args->database) != 0) {
shellFreeTbnames();
return;
}
fprintf(stdout, "total %d tables will be checked by %d threads\n", tbNum, _args->threadNum);
shellRunCheckThreads(con, _args);
int64_t end = taosGetTimestampMs();
fprintf(stdout, "total %d tables checked, failed:%d, time spent %.2f seconds\n", checkedNum, errorNum,
(end - start) / 1000.0);
}

498
tools/shell/src/backup/shellDarwin.c
View File

@ -1,498 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define __USE_XOPEN
#include "os.h"
#include "shell.h"
#include "shellCommand.h"
#include "tbase64.h"
#include "tscLog.h"
#define OPT_ABORT 1 /* <20>Cabort */
int indicator = 1;
struct termios oldtio;
void insertChar(Command *cmd, char *c, int size);
void printHelp() {
char indent[10] = " ";
printf("taos shell is used to test the TDengine database\n");
printf("%s%s\n", indent, "-h");
printf("%s%s%s\n", indent, indent, "TDengine server IP address to connect. The default host is localhost.");
printf("%s%s\n", indent, "-p");
printf("%s%s%s\n", indent, indent, "The password to use when connecting to the server.");
printf("%s%s\n", indent, "-P");
printf("%s%s%s\n", indent, indent, "The TCP/IP port number to use for the connection");
printf("%s%s\n", indent, "-u");
printf("%s%s%s\n", indent, indent, "The user name to use when connecting to the server.");
printf("%s%s\n", indent, "-c");
printf("%s%s%s\n", indent, indent, "Configuration directory.");
printf("%s%s\n", indent, "-s");
printf("%s%s%s\n", indent, indent, "Commands to run without enter the shell.");
printf("%s%s\n", indent, "-r");
printf("%s%s%s\n", indent, indent, "Output time as unsigned long..");
printf("%s%s\n", indent, "-f");
printf("%s%s%s\n", indent, indent, "Script to run without enter the shell.");
printf("%s%s\n", indent, "-d");
printf("%s%s%s\n", indent, indent, "Database to use when connecting to the server.");
printf("%s%s\n", indent, "-t");
printf("%s%s%s\n", indent, indent, "Time zone of the shell, default is local.");
printf("%s%s\n", indent, "-D");
printf("%s%s%s\n", indent, indent, "Use multi-thread to import all SQL files in the directory separately.");
printf("%s%s\n", indent, "-T");
printf("%s%s%s\n", indent, indent, "Number of threads when using multi-thread to import data.");
exit(EXIT_SUCCESS);
}
char DARWINCLIENT_VERSION[] = "Welcome to the TDengine shell from %s, Client Version:%s\n"
"Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n";
char g_password[SHELL_MAX_PASSWORD_LEN];
void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) {
wordexp_t full_path;
for (int i = 1; i < argc; i++) {
// for host
if (strcmp(argv[i], "-h") == 0) {
if (i < argc - 1) {
arguments->host = argv[++i];
} else {
fprintf(stderr, "option -h requires an argument\n");
exit(EXIT_FAILURE);
}
}
// for password
else if ((strncmp(argv[i], "-p", 2) == 0)
|| (strncmp(argv[i], "--password", 10) == 0)) {
strcpy(tsOsName, "Darwin");
printf(DARWINCLIENT_VERSION, tsOsName, taos_get_client_info());
if ((strlen(argv[i]) == 2)
|| (strncmp(argv[i], "--password", 10) == 0)) {
printf("Enter password: ");
taosSetConsoleEcho(false);
if (scanf("%s", g_password) > 1) {
fprintf(stderr, "password read error\n");
}
taosSetConsoleEcho(true);
getchar();
} else {
tstrncpy(g_password, (char *)(argv[i] + 2), SHELL_MAX_PASSWORD_LEN);
}
arguments->password = g_password;
arguments->is_use_passwd = true;
strcpy(argv[i], "");
argc -= 1;
}
// for management port
else if (strcmp(argv[i], "-P") == 0) {
if (i < argc - 1) {
arguments->port = atoi(argv[++i]);
} else {
fprintf(stderr, "option -P requires an argument\n");
exit(EXIT_FAILURE);
}
}
// for user
else if (strcmp(argv[i], "-u") == 0) {
if (i < argc - 1) {
arguments->user = argv[++i];
} else {
fprintf(stderr, "option -u requires an argument\n");
exit(EXIT_FAILURE);
}
} else if (strcmp(argv[i], "-c") == 0) {
if (i < argc - 1) {
if (strlen(argv[++i]) >= TSDB_FILENAME_LEN) {
fprintf(stderr, "config file path: %s overflow max len %d\n", argv[i], TSDB_FILENAME_LEN - 1);
exit(EXIT_FAILURE);
}
strcpy(configDir, argv[i]);
} else {
fprintf(stderr, "Option -c requires an argument\n");
exit(EXIT_FAILURE);
}
} else if (strcmp(argv[i], "-s") == 0) {
if (i < argc - 1) {
arguments->commands = argv[++i];
} else {
fprintf(stderr, "option -s requires an argument\n");
exit(EXIT_FAILURE);
}
} else if (strcmp(argv[i], "-r") == 0) {
arguments->is_raw_time = true;
}
// For temperory batch commands to run TODO
else if (strcmp(argv[i], "-f") == 0) {
if (i < argc - 1) {
strcpy(arguments->file, argv[++i]);
} else {
fprintf(stderr, "option -f requires an argument\n");
exit(EXIT_FAILURE);
}
}
// for default database
else if (strcmp(argv[i], "-d") == 0) {
if (i < argc - 1) {
arguments->database = argv[++i];
} else {
fprintf(stderr, "option -d requires an argument\n");
exit(EXIT_FAILURE);
}
}
// For time zone
else if (strcmp(argv[i], "-t") == 0) {
if (i < argc - 1) {
arguments->timezone = argv[++i];
} else {
fprintf(stderr, "option -t requires an argument\n");
exit(EXIT_FAILURE);
}
}
// For import directory
else if (strcmp(argv[i], "-D") == 0) {
if (i < argc - 1) {
if (wordexp(argv[++i], &full_path, 0) != 0) {
fprintf(stderr, "Invalid path %s\n", argv[i]);
exit(EXIT_FAILURE);
}
strcpy(arguments->dir, full_path.we_wordv[0]);
wordfree(&full_path);
} else {
fprintf(stderr, "option -D requires an argument\n");
exit(EXIT_FAILURE);
}
}
// For time zone
else if (strcmp(argv[i], "-T") == 0) {
if (i < argc - 1) {
arguments->threadNum = atoi(argv[++i]);
} else {
fprintf(stderr, "option -T requires an argument\n");
exit(EXIT_FAILURE);
}
}
// For temperory command TODO
else if (strcmp(argv[i], "--help") == 0) {
printHelp();
exit(EXIT_FAILURE);
} else {
fprintf(stderr, "wrong options\n");
printHelp();
exit(EXIT_FAILURE);
}
}
}
int32_t shellReadCommand(TAOS *con, char *command) {
unsigned hist_counter = history.hend;
char utf8_array[10] = "\0";
Command cmd;
memset(&cmd, 0, sizeof(cmd));
cmd.buffer = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE);
cmd.command = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE);
showOnScreen(&cmd);
// Read input.
char c;
while (1) {
c = getchar();
if (c < 0) { // For UTF-8
int count = countPrefixOnes(c);
utf8_array[0] = c;
for (int k = 1; k < count; k++) {
c = getchar();
utf8_array[k] = c;
}
insertChar(&cmd, utf8_array, count);
} else if (c < '\033') {
// Ctrl keys. TODO: Implement ctrl combinations
switch (c) {
case 1: // ctrl A
positionCursorHome(&cmd);
break;
case 3:
printf("\n");
resetCommand(&cmd, "");
kill(0, SIGINT);
break;
case 4: // EOF or Ctrl+D
printf("\n");
taos_close(con);
// write the history
write_history();
exitShell();
break;
case 5: // ctrl E
positionCursorEnd(&cmd);
break;
case 8:
backspaceChar(&cmd);
break;
case '\n':
case '\r':
printf("\n");
if (isReadyGo(&cmd)) {
sprintf(command, "%s%s", cmd.buffer, cmd.command);
taosMemoryFreeClear(cmd.buffer);
taosMemoryFreeClear(cmd.command);
return 0;
} else {
updateBuffer(&cmd);
}
break;
case 11: // Ctrl + K;
clearLineAfter(&cmd);
break;
case 12: // Ctrl + L;
system("clear");
showOnScreen(&cmd);
break;
case 21: // Ctrl + U
clearLineBefore(&cmd);
break;
}
} else if (c == '\033') {
c = getchar();
switch (c) {
case '[':
c = getchar();
switch (c) {
case 'A': // Up arrow
if (hist_counter != history.hstart) {
hist_counter = (hist_counter + MAX_HISTORY_SIZE - 1) % MAX_HISTORY_SIZE;
resetCommand(&cmd, (history.hist[hist_counter] == NULL) ? "" : history.hist[hist_counter]);
}
break;
case 'B': // Down arrow
if (hist_counter != history.hend) {
int next_hist = (hist_counter + 1) % MAX_HISTORY_SIZE;
if (next_hist != history.hend) {
resetCommand(&cmd, (history.hist[next_hist] == NULL) ? "" : history.hist[next_hist]);
} else {
resetCommand(&cmd, "");
}
hist_counter = next_hist;
}
break;
case 'C': // Right arrow
moveCursorRight(&cmd);
break;
case 'D': // Left arrow
moveCursorLeft(&cmd);
break;
case '1':
if ((c = getchar()) == '~') {
// Home key
positionCursorHome(&cmd);
}
break;
case '2':
if ((c = getchar()) == '~') {
// Insert key
}
break;
case '3':
if ((c = getchar()) == '~') {
// Delete key
deleteChar(&cmd);
}
break;
case '4':
if ((c = getchar()) == '~') {
// End key
positionCursorEnd(&cmd);
}
break;
case '5':
if ((c = getchar()) == '~') {
// Page up key
}
break;
case '6':
if ((c = getchar()) == '~') {
// Page down key
}
break;
case 72:
// Home key
positionCursorHome(&cmd);
break;
case 70:
// End key
positionCursorEnd(&cmd);
break;
}
break;
}
} else if (c == 0x7f) {
// press delete key
backspaceChar(&cmd);
} else {
insertChar(&cmd, &c, 1);
}
}
return 0;
}
void *shellLoopQuery(void *arg) {
if (indicator) {
getOldTerminalMode();
indicator = 0;
}
TAOS *con = (TAOS *)arg;
setThreadName("shellLoopQuery");
taosThreadCleanupPush(cleanup_handler, NULL);
char *command = taosMemoryMalloc(MAX_COMMAND_SIZE);
if (command == NULL){
tscError("failed to malloc command");
return NULL;
}
int32_t err = 0;
do {
// Read command from shell.
memset(command, 0, MAX_COMMAND_SIZE);
setTerminalMode();
err = shellReadCommand(con, command);
if (err) {
break;
}
resetTerminalMode();
} while (shellRunCommand(con, command) == 0);
taosMemoryFreeClear(command);
exitShell();
taosThreadCleanupPop(1);
return NULL;
}
void get_history_path(char *history) { sprintf(history, "%s/%s", getpwuid(getuid())->pw_dir, HISTORY_FILE); }
void clearScreen(int ecmd_pos, int cursor_pos) {
struct winsize w;
if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) {
//fprintf(stderr, "No stream device, and use default value(col 120, row 30)\n");
w.ws_col = 120;
w.ws_row = 30;
}
int cursor_x = cursor_pos / w.ws_col;
int cursor_y = cursor_pos % w.ws_col;
int command_x = ecmd_pos / w.ws_col;
positionCursor(cursor_y, LEFT);
positionCursor(command_x - cursor_x, DOWN);
fprintf(stdout, "\033[2K");
for (int i = 0; i < command_x; i++) {
positionCursor(1, UP);
fprintf(stdout, "\033[2K");
}
fflush(stdout);
}
void showOnScreen(Command *cmd) {
struct winsize w;
if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) {
//fprintf(stderr, "No stream device\n");
w.ws_col = 120;
w.ws_row = 30;
}
TdWchar wc;
int size = 0;
// Print out the command.
char *total_string = taosMemoryMalloc(MAX_COMMAND_SIZE);
memset(total_string, '\0', MAX_COMMAND_SIZE);
if (strcmp(cmd->buffer, "") == 0) {
sprintf(total_string, "%s%s", PROMPT_HEADER, cmd->command);
} else {
sprintf(total_string, "%s%s", CONTINUE_PROMPT, cmd->command);
}
int remain_column = w.ws_col;
/* size = cmd->commandSize + prompt_size; */
for (char *str = total_string; size < cmd->commandSize + prompt_size;) {
int ret = taosMbToWchar(&wc, str, MB_CUR_MAX);
if (ret < 0) break;
size += ret;
/* assert(size >= 0); */
int width = taosWcharWidth(wc);
if (remain_column > width) {
printf("%lc", wc);
remain_column -= width;
} else {
if (remain_column == width) {
printf("%lc\n\r", wc);
remain_column = w.ws_col;
} else {
printf("\n\r%lc", wc);
remain_column = w.ws_col - width;
}
}
str = total_string + size;
}
taosMemoryFree(total_string);
/* for (int i = 0; i < size; i++){ */
/* char c = total_string[i]; */
/* if (k % w.ws_col == 0) { */
/* printf("%c\n\r", c); */
/* } */
/* else { */
/* printf("%c", c); */
/* } */
/* k += 1; */
/* } */
// Position the cursor
int cursor_pos = cmd->screenOffset + prompt_size;
int ecmd_pos = cmd->endOffset + prompt_size;
int cursor_x = cursor_pos / w.ws_col;
int cursor_y = cursor_pos % w.ws_col;
// int cursor_y = cursor % w.ws_col;
int command_x = ecmd_pos / w.ws_col;
int command_y = ecmd_pos % w.ws_col;
// int command_y = (command.size() + prompt_size) % w.ws_col;
positionCursor(command_y, LEFT);
positionCursor(command_x, UP);
positionCursor(cursor_x, DOWN);
positionCursor(cursor_y, RIGHT);
fflush(stdout);
}
void cleanup_handler(void *arg) { resetTerminalMode(); }
void exitShell() {
resetTerminalMode();
exit(EXIT_SUCCESS);
}

279
tools/shell/src/backup/shellImport.c
View File

@ -1,279 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _GNU_SOURCE
#define _XOPEN_SOURCE
#define _DEFAULT_SOURCE
#include "os.h"
#include "shell.h"
#include "shellCommand.h"
#include "tglobal.h"
#include "tutil.h"
static char **shellSQLFiles = NULL;
static int32_t shellSQLFileNum = 0;
static char shellTablesSQLFile[TSDB_FILENAME_LEN] = {0};
typedef struct {
TdThread threadID;
int threadIndex;
int totalThreads;
void *taos;
} ShellThreadObj;
static int shellGetFilesNum(const char *directoryName, const char *prefix)
{
char cmd[1024] = { 0 };
sprintf(cmd, "ls %s/*.%s | wc -l ", directoryName, prefix);
char buf[1024] = { 0 };
if (taosSystem(cmd, buf, sizeof(buf)) < 0) {
fprintf(stderr, "ERROR: failed to execute:%s, error:%s\n", cmd, strerror(errno));
exit(0);
}
int fileNum = 0;
if (sscanf(buf, "%d", &fileNum) != 1) {
fprintf(stderr, "ERROR: failed to execute:%s, parse result error\n", cmd);
exit(0);
}
if (fileNum <= 0) {
fprintf(stderr, "ERROR: directory:%s is empry\n", directoryName);
exit(0);
}
return fileNum;
}
static void shellParseDirectory(const char *directoryName, const char *prefix, char **fileArray, int totalFiles)
{
char cmd[1024] = { 0 };
sprintf(cmd, "ls %s/*.%s | sort", directoryName, prefix);
char buf[1024] = { 0 };
if (taosSystem(cmd, buf, sizeof(buf)) < 0) {
fprintf(stderr, "ERROR: failed to execute:%s, error:%s\n", cmd, strerror(errno));
exit(0);
}
int fileNum = 0;
while (sscanf(buf, "%128s", fileArray[fileNum++])) {
if (strcmp(fileArray[fileNum-1], shellTablesSQLFile) == 0) {
fileNum--;
}
if (fileNum >= totalFiles) {
break;
}
}
if (fileNum != totalFiles) {
fprintf(stderr, "ERROR: directory:%s changed while read\n", directoryName);
exit(0);
}
}
static void shellCheckTablesSQLFile(const char *directoryName)
{
sprintf(shellTablesSQLFile, "%s/tables.sql", directoryName);
if (taosFStatFile(shellTablesSQLFile, NULL, NULL) < 0) {
shellTablesSQLFile[0] = 0;
}
}
static void shellMallocSQLFiles()
{
shellSQLFiles = (char**)taosMemoryCalloc(shellSQLFileNum, sizeof(char*));
for (int i = 0; i < shellSQLFileNum; i++) {
shellSQLFiles[i] = taosMemoryCalloc(1, TSDB_FILENAME_LEN);
}
}
static void shellGetDirectoryFileList(char *inputDir)
{
if (!taosDirExist(inputDir)) {
fprintf(stderr, "ERROR: %s not exist\n", inputDir);
exit(0);
}
if (taosIsDir(inputDir)) {
shellCheckTablesSQLFile(inputDir);
shellSQLFileNum = shellGetFilesNum(inputDir, "sql");
int totalSQLFileNum = shellSQLFileNum;
if (shellTablesSQLFile[0] != 0) {
shellSQLFileNum--;
}
shellMallocSQLFiles();
shellParseDirectory(inputDir, "sql", shellSQLFiles, shellSQLFileNum);
fprintf(stdout, "\nstart to dispose %d files in %s\n", totalSQLFileNum, inputDir);
}
else {
fprintf(stderr, "ERROR: %s is not a directory\n", inputDir);
exit(0);
}
}
static void shellSourceFile(TAOS *con, char *fptr) {
wordexp_t full_path;
int read_len = 0;
char * cmd = taosMemoryMalloc(tsMaxSQLStringLen);
size_t cmd_len = 0;
char * line = NULL;
if (wordexp(fptr, &full_path, 0) != 0) {
fprintf(stderr, "ERROR: illegal file name\n");
taosMemoryFree(cmd);
return;
}
char *fname = full_path.we_wordv[0];
if (fname == NULL) {
fprintf(stderr, "ERROR: invalid filename\n");
taosMemoryFree(cmd);
return;
}
/*
if (access(fname, F_OK) != 0) {
fprintf(stderr, "ERROR: file %s is not exist\n", fptr);
wordfree(&full_path);
taosMemoryFree(cmd);
return;
}
if (access(fname, R_OK) != 0) {
fprintf(stderr, "ERROR: file %s is not readable\n", fptr);
wordfree(&full_path);
taosMemoryFree(cmd);
return;
}
*/
// FILE *f = fopen(fname, "r");
TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ | TD_FILE_STREAM);
if (pFile == NULL) {
fprintf(stderr, "ERROR: failed to open file %s\n", fname);
wordfree(&full_path);
taosMemoryFree(cmd);
return;
}
fprintf(stdout, "begin import file:%s\n", fname);
int lineNo = 0;
while ((read_len = taosGetLineFile(pFile, &line)) != -1) {
++lineNo;
if (read_len >= tsMaxSQLStringLen) continue;
line[--read_len] = '\0';
if (read_len == 0 || isCommentLine(line)) { // line starts with #
continue;
}
if (line[read_len - 1] == '\\') {
line[read_len - 1] = ' ';
memcpy(cmd + cmd_len, line, read_len);
cmd_len += read_len;
continue;
}
memcpy(cmd + cmd_len, line, read_len);
TAOS_RES* pSql = taos_query(con, cmd);
int32_t code = taos_errno(pSql);
if (code != 0) {
fprintf(stderr, "DB error: %s: %s (%d)\n", taos_errstr(pSql), fname, lineNo);
}
/* free local resouce: allocated memory/metric-meta refcnt */
taos_free_result(pSql);
memset(cmd, 0, MAX_COMMAND_SIZE);
cmd_len = 0;
}
taosMemoryFree(cmd);
if(line != NULL) taosMemoryFree(line);
wordfree(&full_path);
taosCloseFile(&pFile);
}
void* shellImportThreadFp(void *arg)
{
ShellThreadObj *pThread = (ShellThreadObj*)arg;
setThreadName("shellImportThrd");
for (int f = 0; f < shellSQLFileNum; ++f) {
if (f % pThread->totalThreads == pThread->threadIndex) {
char *SQLFileName = shellSQLFiles[f];
shellSourceFile(pThread->taos, SQLFileName);
}
}
return NULL;
}
static void shellRunImportThreads(SShellArguments* _args)
{
TdThreadAttr thattr;
ShellThreadObj *threadObj = (ShellThreadObj *)taosMemoryCalloc(_args->threadNum, sizeof(ShellThreadObj));
for (int t = 0; t < _args->threadNum; ++t) {
ShellThreadObj *pThread = threadObj + t;
pThread->threadIndex = t;
pThread->totalThreads = _args->threadNum;
pThread->taos = taos_connect(_args->host, _args->user, _args->password, _args->database, tsDnodeShellPort);
if (pThread->taos == NULL) {
fprintf(stderr, "ERROR: thread:%d failed connect to TDengine, error:%s\n", pThread->threadIndex, "null taos"/*taos_errstr(pThread->taos)*/);
exit(0);
}
taosThreadAttrInit(&thattr);
taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&(pThread->threadID), &thattr, shellImportThreadFp, (void*)pThread) != 0) {
fprintf(stderr, "ERROR: thread:%d failed to start\n", pThread->threadIndex);
exit(0);
}
}
for (int t = 0; t < _args->threadNum; ++t) {
taosThreadJoin(threadObj[t].threadID, NULL);
}
for (int t = 0; t < _args->threadNum; ++t) {
taos_close(threadObj[t].taos);
}
taosMemoryFree(threadObj);
}
void source_dir(TAOS* con, SShellArguments* _args) {
shellGetDirectoryFileList(_args->dir);
int64_t start = taosGetTimestampMs();
if (shellTablesSQLFile[0] != 0) {
shellSourceFile(con, shellTablesSQLFile);
int64_t end = taosGetTimestampMs();
fprintf(stdout, "import %s finished, time spent %.2f seconds\n", shellTablesSQLFile, (end - start) / 1000.0);
}
shellRunImportThreads(_args);
int64_t end = taosGetTimestampMs();
fprintf(stdout, "import %s finished, time spent %.2f seconds\n", _args->dir, (end - start) / 1000.0);
}

325
tools/shell/src/backup/shellWindows.c
View File

@ -1,325 +0,0 @@
/*******************************************************************
* Copyright (c) 2017 by TAOS Technologies, Inc.
* All rights reserved.
*
* This file is proprietary and confidential to TAOS Technologies.
* No part of this file may be reproduced, stored, transmitted,
* disclosed or used in any form or by any means other than as
* expressly provided by the written permission from Jianhui Tao
*
* ****************************************************************/
#include <assert.h>
#include <regex.h>
#include <stdio.h>
#include "../../../../include/client/taos.h"
#include "os.h"
#include "shell.h"
#include "shellCommand.h"
extern char configDir[];
char WINCLIENT_VERSION[] = "Welcome to the TDengine shell from %s, Client Version:%s\n"
"Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n";
void printVersion() {
printf("version: %s\n", version);
}
void printHelp() {
char indent[10] = " ";
printf("taos shell is used to test the TDengine database\n");
printf("%s%s\n", indent, "-h");
printf("%s%s%s\n", indent, indent, "TDengine server FQDN to connect. The default host is localhost.");
printf("%s%s\n", indent, "-p");
printf("%s%s%s\n", indent, indent, "The password to use when connecting to the server.");
printf("%s%s\n", indent, "-P");
printf("%s%s%s\n", indent, indent, "The TCP/IP port number to use for the connection");
printf("%s%s\n", indent, "-u");
printf("%s%s%s\n", indent, indent, "The user name to use when connecting to the server.");
printf("%s%s\n", indent, "-A");
printf("%s%s%s\n", indent, indent, "The user auth to use when connecting to the server.");
printf("%s%s\n", indent, "-c");
printf("%s%s%s\n", indent, indent, "Configuration directory.");
printf("%s%s\n", indent, "-C");
printf("%s%s%s\n", indent, indent, "Dump configuration.");
printf("%s%s\n", indent, "-s");
printf("%s%s%s\n", indent, indent, "Commands to run without enter the shell.");
printf("%s%s\n", indent, "-r");
printf("%s%s%s\n", indent, indent, "Output time as unsigned long..");
printf("%s%s\n", indent, "-f");
printf("%s%s%s\n", indent, indent, "Script to run without enter the shell.");
printf("%s%s\n", indent, "-d");
printf("%s%s%s\n", indent, indent, "Database to use when connecting to the server.");
printf("%s%s\n", indent, "-t");
printf("%s%s%s\n", indent, indent, "Time zone of the shell, default is local.");
printf("%s%s\n", indent, "-n");
printf("%s%s%s\n", indent, indent, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup|sync|speed|fqdn.");
printf("%s%s\n", indent, "-l");
printf("%s%s%s\n", indent, indent, "Packet length used for net test, default is 1000 bytes.");
printf("%s%s\n", indent, "-N");
printf("%s%s%s\n", indent, indent, "Packet numbers used for net test, default is 100.");
printf("%s%s\n", indent, "-S");
printf("%s%s%s\n", indent, indent, "Packet type used for net test, default is TCP.");
printf("%s%s\n", indent, "-V");
printf("%s%s%s\n", indent, indent, "Print program version.");
exit(EXIT_SUCCESS);
}
char g_password[SHELL_MAX_PASSWORD_LEN];
void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) {
for (int i = 1; i < argc; i++) {
// for host
if (strcmp(argv[i], "-h") == 0) {
if (i < argc - 1) {
arguments->host = argv[++i];
} else {
fprintf(stderr, "option -h requires an argument\n");
exit(EXIT_FAILURE);
}
}
// for password
else if ((strncmp(argv[i], "-p", 2) == 0)
|| (strncmp(argv[i], "--password", 10) == 0)) {
arguments->is_use_passwd = true;
strcpy(tsOsName, "Windows");
printf(WINCLIENT_VERSION, tsOsName, taos_get_client_info());
if ((strlen(argv[i]) == 2)
|| (strncmp(argv[i], "--password", 10) == 0)) {
printf("Enter password: ");
taosSetConsoleEcho(false);
if (scanf("%s", g_password) > 1) {
fprintf(stderr, "password read error!\n");
}
taosSetConsoleEcho(true);
getchar();
} else {
tstrncpy(g_password, (char *)(argv[i] + 2), SHELL_MAX_PASSWORD_LEN);
}
arguments->password = g_password;
strcpy(argv[i], "");
argc -= 1;
}
// for management port
else if (strcmp(argv[i], "-P") == 0) {
if (i < argc - 1) {
arguments->port = atoi(argv[++i]);
} else {
fprintf(stderr, "option -P requires an argument\n");
exit(EXIT_FAILURE);
}
}
// for user
else if (strcmp(argv[i], "-u") == 0) {
if (i < argc - 1) {
arguments->user = argv[++i];
} else {
fprintf(stderr, "option -u requires an argument\n");
exit(EXIT_FAILURE);
}
} else if (strcmp(argv[i], "-A") == 0) {
if (i < argc - 1) {
arguments->auth = argv[++i];
} else {
fprintf(stderr, "option -A requires an argument\n");
exit(EXIT_FAILURE);
}
} else if (strcmp(argv[i], "-c") == 0) {
if (i < argc - 1) {
char *tmp = argv[++i];
if (strlen(tmp) >= TSDB_FILENAME_LEN) {
fprintf(stderr, "config file path: %s overflow max len %d\n", tmp, TSDB_FILENAME_LEN - 1);
exit(EXIT_FAILURE);
}
strcpy(configDir, tmp);
} else {
fprintf(stderr, "Option -c requires an argument\n");
exit(EXIT_FAILURE);
}
} else if (strcmp(argv[i], "-C") == 0) {
arguments->dump_config = true;
} else if (strcmp(argv[i], "-s") == 0) {
if (i < argc - 1) {
arguments->commands = argv[++i];
} else {
fprintf(stderr, "option -s requires an argument\n");
exit(EXIT_FAILURE);
}
} else if (strcmp(argv[i], "-r") == 0) {
arguments->is_raw_time = true;
}
// For temperory batch commands to run TODO
else if (strcmp(argv[i], "-f") == 0) {
if (i < argc - 1) {
strcpy(arguments->file, argv[++i]);
} else {
fprintf(stderr, "option -f requires an argument\n");
exit(EXIT_FAILURE);
}
}
// for default database
else if (strcmp(argv[i], "-d") == 0) {
if (i < argc - 1) {
arguments->database = argv[++i];
} else {
fprintf(stderr, "option -d requires an argument\n");
exit(EXIT_FAILURE);
}
}
// For time zone
else if (strcmp(argv[i], "-t") == 0) {
if (i < argc - 1) {
arguments->timezone = argv[++i];
} else {
fprintf(stderr, "option -t requires an argument\n");
exit(EXIT_FAILURE);
}
}
else if (strcmp(argv[i], "-n") == 0) {
if (i < argc - 1) {
arguments->netTestRole = argv[++i];
} else {
fprintf(stderr, "option -n requires an argument\n");
exit(EXIT_FAILURE);
}
}
else if (strcmp(argv[i], "-l") == 0) {
if (i < argc - 1) {
arguments->pktLen = atoi(argv[++i]);
} else {
fprintf(stderr, "option -l requires an argument\n");
exit(EXIT_FAILURE);
}
}
else if (strcmp(argv[i], "-N") == 0) {
if (i < argc - 1) {
arguments->pktNum = atoi(argv[++i]);
} else {
fprintf(stderr, "option -N requires an argument\n");
exit(EXIT_FAILURE);
}
}
else if (strcmp(argv[i], "-S") == 0) {
if (i < argc - 1) {
arguments->pktType = argv[++i];
} else {
fprintf(stderr, "option -S requires an argument\n");
exit(EXIT_FAILURE);
}
}
else if (strcmp(argv[i], "-V") == 0) {
printVersion();
exit(EXIT_SUCCESS);
}
// For temperory command TODO
else if (strcmp(argv[i], "--help") == 0) {
printHelp();
exit(EXIT_SUCCESS);
} else {
fprintf(stderr, "wrong options\n");
printHelp();
exit(EXIT_FAILURE);
}
}
}
void shellPrintContinuePrompt() { printf("%s", CONTINUE_PROMPT); }
void shellPrintPrompt() { printf("%s", PROMPT_HEADER); }
void updateBuffer(Command *cmd) {
if (regex_match(cmd->buffer, "(\\s+$)|(^$)", REG_EXTENDED)) strcat(cmd->command, " ");
strcat(cmd->buffer, cmd->command);
memset(cmd->command, 0, MAX_COMMAND_SIZE);
cmd->cursorOffset = 0;
}
int isReadyGo(Command *cmd) {
char *total = taosMemoryMalloc(MAX_COMMAND_SIZE);
memset(total, 0, MAX_COMMAND_SIZE);
sprintf(total, "%s%s", cmd->buffer, cmd->command);
char *reg_str =
"(^.*;\\s*$)|(^\\s*$)|(^\\s*exit\\s*$)|(^\\s*q\\s*$)|(^\\s*quit\\s*$)|(^"
"\\s*clear\\s*$)";
if (regex_match(total, reg_str, REG_EXTENDED | REG_ICASE)) {
taosMemoryFree(total);
return 1;
}
taosMemoryFree(total);
return 0;
}
void insertChar(Command *cmd, char c) {
// TODO: Check if the length enough.
if (cmd->cursorOffset >= MAX_COMMAND_SIZE) {
fprintf(stdout, "sql is larger than %d bytes", MAX_COMMAND_SIZE);
return;
}
cmd->command[cmd->cursorOffset++] = c;
}
int32_t shellReadCommand(TAOS *con, char command[]) {
Command cmd;
memset(&cmd, 0, sizeof(cmd));
cmd.buffer = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE);
cmd.command = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE);
// Read input.
char c;
while (1) {
c = getchar();
switch (c) {
case '\n':
case '\r':
if (isReadyGo(&cmd)) {
sprintf(command, "%s%s", cmd.buffer, cmd.command);
taosMemoryFree(cmd.buffer);
cmd.buffer = NULL;
taosMemoryFree(cmd.command);
cmd.command = NULL;
return 0;
} else {
shellPrintContinuePrompt();
updateBuffer(&cmd);
}
break;
default:
insertChar(&cmd, c);
}
}
return 0;
}
void *shellLoopQuery(void *arg) {
TAOS *con = (TAOS *)arg;
char *command = taosMemoryMalloc(MAX_COMMAND_SIZE);
if (command == NULL) return NULL;
int32_t err = 0;
do {
memset(command, 0, MAX_COMMAND_SIZE);
shellPrintPrompt();
// Read command from shell.
err = shellReadCommand(con, command);
if (err) {
break;
}
} while (shellRunCommand(con, command) == 0);
return NULL;
}
void get_history_path(char *history) { sprintf(history, "C:/TDengine/%s", HISTORY_FILE); }
void exitShell() { exit(EXIT_SUCCESS); }

29
tools/shell/src/backup/tnettest.h
View File

@ -1,29 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TNETTEST_H
#define TDENGINE_TNETTEST_H
#ifdef __cplusplus
extern "C" {
#endif
void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, int32_t pkgNum, char *pkgType);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_TNETTEST_H

354
tools/shell/src/shellArguments.c Normal file
View File

@ -0,0 +1,354 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "shellInt.h"
#define SHELL_HOST "The auth string to use when connecting to the server."
#define SHELL_PORT "The TCP/IP port number to use for the connection."
#define SHELL_USER "The user name to use when connecting to the server."
#define SHELL_PASSWORD "The password to use when connecting to the server."
#define SHELL_AUTH "The auth string to use when connecting to the server."
#define SHELL_GEN_AUTH "Generate auth string from password."
#define SHELL_CFG_DIR "Configuration directory."
#define SHELL_DMP_CFG "Dump configuration."
#define SHELL_CMD "Commands to run without enter the shell."
#define SHELL_RAW_TIME "Output time as uint64_t."
#define SHELL_FILE "Script to run without enter the shell."
#define SHELL_DB "Database to use when connecting to the server."
#define SHELL_CHECK "Check the service status."
#define SHELL_STARTUP "Check the details of the service status."
#define SHELL_WIDTH "Set the default binary display width, default is 30."
#define SHELL_NET_ROLE "Net role when network connectivity test, options: client|server."
#define SHELL_PKG_LEN "Packet length used for net test, default is 1024 bytes."
#define SHELL_PKT_NUM "Packet numbers used for net test, default is 100."
#define SHELL_VERSION "Print program version."
#define SHELL_EMAIL "<support@taosdata.com>"
void shellPrintHelp() {
char indent[] = " ";
printf("Usage: taos [OPTION...] \n\n");
printf("%s%s%s%s\n", indent, "-a,", indent, SHELL_AUTH);
printf("%s%s%s%s\n", indent, "-A,", indent, SHELL_GEN_AUTH);
printf("%s%s%s%s\n", indent, "-c,", indent, SHELL_CFG_DIR);
printf("%s%s%s%s\n", indent, "-C,", indent, SHELL_DMP_CFG);
printf("%s%s%s%s\n", indent, "-d,", indent, SHELL_DB);
printf("%s%s%s%s\n", indent, "-f,", indent, SHELL_FILE);
printf("%s%s%s%s\n", indent, "-h,", indent, SHELL_HOST);
printf("%s%s%s%s\n", indent, "-k,", indent, SHELL_CHECK);
printf("%s%s%s%s\n", indent, "-l,", indent, SHELL_PKG_LEN);
printf("%s%s%s%s\n", indent, "-n,", indent, SHELL_NET_ROLE);
printf("%s%s%s%s\n", indent, "-N,", indent, SHELL_PKT_NUM);
printf("%s%s%s%s\n", indent, "-p,", indent, SHELL_PASSWORD);
printf("%s%s%s%s\n", indent, "-P,", indent, SHELL_PORT);
printf("%s%s%s%s\n", indent, "-r,", indent, SHELL_RAW_TIME);
printf("%s%s%s%s\n", indent, "-s,", indent, SHELL_CMD);
printf("%s%s%s%s\n", indent, "-t,", indent, SHELL_STARTUP);
printf("%s%s%s%s\n", indent, "-u,", indent, SHELL_USER);
printf("%s%s%s%s\n", indent, "-w,", indent, SHELL_WIDTH);
printf("%s%s%s%s\n", indent, "-V,", indent, SHELL_VERSION);
printf("\n\nReport bugs to %s.\n", SHELL_EMAIL);
}
static int32_t shellParseSingleOpt(int32_t key, char *arg) {
SShellArgs *pArgs = &shell.args;
switch (key) {
case 'h':
pArgs->host = arg;
break;
case 'P':
pArgs->port = atoi(arg);
break;
case 'u':
pArgs->user = arg;
break;
case 'p':
break;
case 'a':
pArgs->auth = arg;
break;
case 'A':
pArgs->is_gen_auth = true;
break;
case 'c':
pArgs->cfgdir = arg;
break;
case 'C':
pArgs->is_dump_config = true;
break;
case 's':
pArgs->commands = arg;
break;
case 'r':
pArgs->is_raw_time = true;
break;
case 'f':
tstrncpy(pArgs->file, arg, sizeof(pArgs->file));
break;
case 'd':
pArgs->database = arg;
break;
case 'k':
pArgs->is_check = true;
break;
case 't':
pArgs->is_startup = true;
break;
case 'w':
pArgs->displayWidth = atoi(arg);
break;
case 'n':
pArgs->netrole = arg;
break;
case 'l':
pArgs->pktLen = atoi(arg);
break;
case 'N':
pArgs->pktNum = atoi(arg);
break;
case 'V':
pArgs->is_version = true;
break;
case '?':
pArgs->is_help = true;
break;
case 1:
pArgs->abort = 1;
break;
default:
return ARGP_ERR_UNKNOWN;
}
return 0;
}
int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) {
SShellArgs *pArgs = &shell.args;
for (int i = 1; i < argc; i++) {
if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "--usage") == 0 || strcmp(argv[i], "-?") == 0) {
shellParseSingleOpt('?', NULL);
return 0;
}
char *key = argv[i];
int32_t keyLen = strlen(key);
if (keyLen != 2) {
fprintf(stderr, "invalid option %s\n", key);
return -1;
}
if (key[0] != '-') {
fprintf(stderr, "invalid option %s\n", key);
return -1;
}
if (key[1] == 'h' || key[1] == 'P' || key[1] == 'u' || key[1] == 'a' || key[1] == 'c' || key[1] == 's' ||
key[1] == 'f' || key[1] == 'd' || key[1] == 'w' || key[1] == 'n' || key[1] == 'l' || key[1] == 'N') {
if (i + 1 >= argc) {
fprintf(stderr, "option %s requires an argument\n", key);
return -1;
}
char *val = argv[i + 1];
if (val[0] == '-') {
fprintf(stderr, "option %s requires an argument\n", key);
return -1;
}
shellParseSingleOpt(key[1], val);
i++;
} else if (key[1] == 'p' || key[1] == 'A' || key[1] == 'c' || key[1] == 'r' || key[1] == 'k' || key[1] == 't' ||
key[1] == 'V') {
shellParseSingleOpt(key[1], NULL);
} else {
fprintf(stderr, "invalid option %s\n", key);
return -1;
}
}
return 0;
}
#ifdef LINUX
#include <argp.h>
#include <termio.h>
const char *argp_program_version = version;
const char *argp_program_bug_address = SHELL_EMAIL;
static struct argp_option shellOptions[] = {
{"host", 'h', "HOST", 0, SHELL_HOST},
{"port", 'P', "PORT", 0, SHELL_PORT},
{"user", 'u', "USER", 0, SHELL_USER},
{0, 'p', 0, 0, SHELL_PASSWORD},
{"auth", 'a', "AUTH", 0, SHELL_AUTH},
{"generate-auth", 'A', 0, 0, SHELL_GEN_AUTH},
{"config-dir", 'c', "DIR", 0, SHELL_CFG_DIR},
{"dump-config", 'C', 0, 0, SHELL_DMP_CFG},
{"commands", 's', "COMMANDS", 0, SHELL_CMD},
{"raw-time", 'r', 0, 0, SHELL_RAW_TIME},
{"file", 'f', "FILE", 0, SHELL_FILE},
{"database", 'd', "DATABASE", 0, SHELL_DB},
{"check", 'k', 0, 0, SHELL_CHECK},
{"startup", 't', 0, 0, SHELL_STARTUP},
{"display-width", 'w', "WIDTH", 0, SHELL_WIDTH},
{"netrole", 'n', "NETROLE", 0, SHELL_NET_ROLE},
{"pktlen", 'l', "PKTLEN", 0, SHELL_PKG_LEN},
{"pktnum", 'N', "PKTNUM", 0, SHELL_PKT_NUM},
{0},
};
static error_t shellParseOpt(int32_t key, char *arg, struct argp_state *state) { return shellParseSingleOpt(key, arg); }
static struct argp shellArgp = {shellOptions, shellParseOpt, "", ""};
static void shellParseArgsUseArgp(int argc, char *argv[]) {
argp_program_version = shell.info.programVersion;
argp_parse(&shellArgp, argc, argv, 0, 0, &shell.args);
}
#endif
static void shellInitArgs(int argc, char *argv[]) {
for (int i = 1; i < argc; i++) {
if (strncmp(argv[i], "-p", 2) == 0) {
printf(shell.info.clientVersion, tsOsName, taos_get_client_info());
if (strlen(argv[i]) == 2) {
printf("Enter password: ");
taosSetConsoleEcho(false);
if (scanf("%20s", shell.args.password) > 1) {
fprintf(stderr, "password reading error\n");
}
taosSetConsoleEcho(true);
if (EOF == getchar()) {
fprintf(stderr, "getchar() return EOF\n");
}
} else {
tstrncpy(shell.args.password, (char *)(argv[i] + 2), sizeof(shell.args.password));
strcpy(argv[i], "-p");
}
}
}
if (strlen(shell.args.password) == 0) {
tstrncpy(shell.args.password, TSDB_DEFAULT_PASS, sizeof(shell.args.password));
}
SShellArgs *pArgs = &shell.args;
pArgs->user = TSDB_DEFAULT_USER;
pArgs->pktLen = SHELL_DEF_PKG_LEN;
pArgs->pktNum = SHELL_DEF_PKG_NUM;
pArgs->displayWidth = SHELL_DEFAULT_MAX_BINARY_DISPLAY_WIDTH;
}
static int32_t shellCheckArgs() {
SShellArgs *pArgs = &shell.args;
if (pArgs->host != NULL && (strlen(pArgs->host) <= 0 || strlen(pArgs->host) > TSDB_FQDN_LEN)) {
printf("Invalid host:%s\n", pArgs->host);
return -1;
}
if (pArgs->user != NULL && (strlen(pArgs->user) <= 0 || strlen(pArgs->user) > TSDB_USER_LEN)) {
printf("Invalid user:%s\n", pArgs->user);
return -1;
}
if (pArgs->auth != NULL && (strlen(pArgs->auth) <= 0 || strlen(pArgs->auth) > TSDB_PASSWORD_LEN)) {
printf("Invalid auth:%s\n", pArgs->auth);
return -1;
}
if (pArgs->database != NULL && (strlen(pArgs->database) <= 0 || strlen(pArgs->database) > TSDB_DB_NAME_LEN)) {
printf("Invalid database:%s\n", pArgs->database);
return -1;
}
if (pArgs->file[0] != 0) {
char fullname[PATH_MAX] = {0};
if (taosExpandDir(pArgs->file, fullname, PATH_MAX) == 0) {
tstrncpy(pArgs->file, fullname, PATH_MAX);
}
}
if (pArgs->cfgdir != NULL) {
if (strlen(pArgs->cfgdir) <= 0 || strlen(pArgs->cfgdir) >= PATH_MAX) {
printf("Invalid cfgdir:%s\n", pArgs->cfgdir);
return -1;
} else {
if (taosExpandDir(pArgs->cfgdir, configDir, PATH_MAX) != 0) {
tstrncpy(configDir, pArgs->cfgdir, PATH_MAX);
}
}
}
if (pArgs->commands != NULL && (strlen(pArgs->commands) <= 0)) {
printf("Invalid commands:%s\n", pArgs->commands);
return -1;
}
if (pArgs->netrole != NULL && !(strcmp(pArgs->netrole, "client") == 0 || strcmp(pArgs->netrole, "server") == 0)) {
printf("Invalid netrole:%s\n", pArgs->netrole);
return -1;
}
if (pArgs->password != NULL && (strlen(pArgs->password) <= 0)) {
printf("Invalid password\n");
return -1;
}
if (pArgs->pktLen < SHELL_MIN_PKG_LEN || pArgs->pktLen > SHELL_MAX_PKG_LEN) {
printf("Invalid pktLen:%d, range:[%d, %d]\n", pArgs->pktLen, SHELL_MIN_PKG_LEN, SHELL_MAX_PKG_LEN);
return -1;
}
if (pArgs->pktNum < SHELL_MIN_PKG_NUM || pArgs->pktNum > SHELL_MAX_PKG_NUM) {
printf("Invalid pktNum:%d, range:[%d, %d]\n", pArgs->pktNum, SHELL_MIN_PKG_NUM, SHELL_MAX_PKG_NUM);
return -1;
}
if (pArgs->displayWidth <= 0 || pArgs->displayWidth > 10 * 1024) {
printf("Invalid displayWidth:%d, range:[1, 10 * 1024]\n", pArgs->displayWidth);
return -1;
}
return 0;
}
int32_t shellParseArgs(int32_t argc, char *argv[]) {
shellInitArgs(argc, argv);
shell.info.clientVersion =
"Welcome to the TDengine shell from %s, Client Version:%s\n"
"Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n";
shell.info.promptHeader = "taos> ";
shell.info.promptContinue = " -> ";
shell.info.promptSize = 6;
snprintf(shell.info.programVersion, sizeof(shell.info.programVersion), "version: %s", version);
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
shell.info.osname = "Windows";
snprintf(shell.history.file, TSDB_FILENAME_LEN, "C:/TDengine/%s", SHELL_HISTORY_FILE);
if (shellParseArgsWithoutArgp(argc, argv) != 0) return -1;
#elif defined(_TD_DARWIN_64)
shell.info.osname = "Darwin";
snprintf(shell.history.file, TSDB_FILENAME_LEN, "%s/%s", getpwuid(getuid())->pw_dir, SHELL_HISTORY_FILE);
if (shellParseArgsWithoutArgp(argc, argv) != 0) return -1;
#else
shell.info.osname = "Linux";
snprintf(shell.history.file, TSDB_FILENAME_LEN, "%s/%s", getenv("HOME"), SHELL_HISTORY_FILE);
shellParseArgsUseArgp(argc, argv);
// if (shellParseArgsWithoutArgp(argc, argv) != 0) return -1;
if (shell.args.abort) {
return -1;
}
#endif
return shellCheckArgs();
}

496
tools/shell/src/shellCommand.c
View File

@ -14,22 +14,122 @@
*/
#define __USE_XOPEN
#include "shellInt.h"
#include "shellCommand.h"
#include "os.h"
#include "shell.h"
#include <regex.h>
#define LEFT 1
#define RIGHT 2
#define UP 3
#define DOWN 4
#define PSIZE shell.info.promptSize
typedef struct {
char widthInString;
char widthOnScreen;
} UTFCodeInfo;
char *buffer;
char *command;
uint32_t commandSize;
uint32_t bufferSize;
uint32_t cursorOffset;
uint32_t screenOffset;
uint32_t endOffset;
} SShellCmd;
int countPrefixOnes(unsigned char c) {
unsigned char mask = 127;
static int32_t shellCountPrefixOnes(uint8_t c);
static void shellGetPrevCharSize(const char *str, int32_t pos, int32_t *size, int32_t *width);
static void shellGetNextCharSize(const char *str, int32_t pos, int32_t *size, int32_t *width);
static void shellInsertChar(SShellCmd *cmd, char *c, int size);
static void shellBackspaceChar(SShellCmd *cmd);
static void shellClearLineBefore(SShellCmd *cmd);
static void shellClearLineAfter(SShellCmd *cmd);
static void shellDeleteChar(SShellCmd *cmd);
static void shellMoveCursorLeft(SShellCmd *cmd);
static void shellMoveCursorRight(SShellCmd *cmd);
static void shellPositionCursorHome(SShellCmd *cmd);
static void shellPositionCursorEnd(SShellCmd *cmd);
static void shellPrintChar(char c, int32_t times);
static void shellPositionCursor(int32_t step, int32_t direction);
static void shellUpdateBuffer(SShellCmd *cmd);
static int32_t shellIsReadyGo(SShellCmd *cmd);
static void shellGetMbSizeInfo(const char *str, int32_t *size, int32_t *width);
static void shellResetCommand(SShellCmd *cmd, const char s[]);
static void shellClearScreen(int32_t ecmd_pos, int32_t cursor_pos);
static void shellShowOnScreen(SShellCmd *cmd);
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
static void shellPrintContinuePrompt() { printf("%s", shell.args.promptContinue); }
static void shellPrintPrompt() { printf("%s", shell.args.promptHeader); }
void shellUpdateBuffer(SShellCmd *cmd) {
if (shellRegexMatch(cmd->buffer, "(\\s+$)|(^$)", REG_EXTENDED)) strcat(cmd->command, " ");
strcat(cmd->buffer, cmd->command);
memset(cmd->command, 0, SHELL_MAX_COMMAND_SIZE);
cmd->cursorOffset = 0;
}
int shellIsReadyGo(SShellCmd *cmd) {
char *total = taosMemoryMalloc(SHELL_MAX_COMMAND_SIZE);
memset(total, 0, SHELL_MAX_COMMAND_SIZE);
sprintf(total, "%s%s", cmd->buffer, cmd->command);
char *reg_str =
"(^.*;\\s*$)|(^\\s*$)|(^\\s*exit\\s*$)|(^\\s*q\\s*$)|(^\\s*quit\\s*$)|(^"
"\\s*clear\\s*$)";
if (shellRegexMatch(total, reg_str, REG_EXTENDED | REG_ICASE)) {
taosMemoryFree(total);
return 1;
}
taosMemoryFree(total);
return 0;
}
void shellInsertChar(SShellCmd *cmd, char c) {
if (cmd->cursorOffset >= SHELL_MAX_COMMAND_SIZE) {
fprintf(stdout, "sql is larger than %d bytes", SHELL_MAX_COMMAND_SIZE);
return;
}
cmd->command[cmd->cursorOffset++] = c;
}
int32_t shellReadCommand(char command[]) {
SShellCmd cmd;
memset(&cmd, 0, sizeof(cmd));
cmd.buffer = (char *)taosMemoryCalloc(1, SHELL_MAX_COMMAND_SIZE);
cmd.command = (char *)taosMemoryCalloc(1, SHELL_MAX_COMMAND_SIZE);
// Read input.
char c;
while (1) {
c = getchar();
switch (c) {
case '\n':
case '\r':
if (shellIsReadyGo(&cmd)) {
sprintf(command, "%s%s", cmd.buffer, cmd.command);
taosMemoryFree(cmd.buffer);
cmd.buffer = NULL;
taosMemoryFree(cmd.command);
cmd.command = NULL;
return 0;
} else {
shellPrintContinuePrompt();
shellUpdateBuffer(&cmd);
}
break;
default:
shellInsertChar(&cmd, c);
}
}
return 0;
}
#else
int32_t shellCountPrefixOnes(uint8_t c) {
uint8_t mask = 127;
mask = ~mask;
int ret = 0;
int32_t ret = 0;
while ((c & mask) != 0) {
ret++;
c <<= 1;
@ -38,7 +138,7 @@ int countPrefixOnes(unsigned char c) {
return ret;
}
void getPrevCharSize(const char *str, int pos, int *size, int *width) {
void shellGetPrevCharSize(const char *str, int32_t pos, int32_t *size, int32_t *width) {
assert(pos > 0);
TdWchar wc;
@ -48,16 +148,16 @@ void getPrevCharSize(const char *str, int pos, int *size, int *width) {
while (--pos >= 0) {
*size += 1;
if (str[pos] > 0 || countPrefixOnes((unsigned char)str[pos]) > 1) break;
if (str[pos] > 0 || shellCountPrefixOnes((uint8_t)str[pos]) > 1) break;
}
int rc = taosMbToWchar(&wc, str + pos, MB_CUR_MAX);
int32_t rc = taosMbToWchar(&wc, str + pos, MB_CUR_MAX);
assert(rc == *size);
*width = taosWcharWidth(wc);
}
void getNextCharSize(const char *str, int pos, int *size, int *width) {
void shellGetNextCharSize(const char *str, int32_t pos, int32_t *size, int32_t *width) {
assert(pos >= 0);
TdWchar wc;
@ -65,13 +165,13 @@ void getNextCharSize(const char *str, int pos, int *size, int *width) {
*width = taosWcharWidth(wc);
}
void insertChar(Command *cmd, char *c, int size) {
void shellInsertChar(SShellCmd *cmd, char *c, int32_t size) {
assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset);
TdWchar wc;
if (taosMbToWchar(&wc, c, size) < 0) return;
clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size);
shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE);
/* update the buffer */
memmove(cmd->command + cmd->cursorOffset + size, cmd->command + cmd->cursorOffset,
cmd->commandSize - cmd->cursorOffset);
@ -81,122 +181,122 @@ void insertChar(Command *cmd, char *c, int size) {
cmd->cursorOffset += size;
cmd->screenOffset += taosWcharWidth(wc);
cmd->endOffset += taosWcharWidth(wc);
showOnScreen(cmd);
shellShowOnScreen(cmd);
}
void backspaceChar(Command *cmd) {
void shellBackspaceChar(SShellCmd *cmd) {
assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset);
if (cmd->cursorOffset > 0) {
clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size);
int size = 0;
int width = 0;
getPrevCharSize(cmd->command, cmd->cursorOffset, &size, &width);
shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE);
int32_t size = 0;
int32_t width = 0;
shellGetPrevCharSize(cmd->command, cmd->cursorOffset, &size, &width);
memmove(cmd->command + cmd->cursorOffset - size, cmd->command + cmd->cursorOffset,
cmd->commandSize - cmd->cursorOffset);
cmd->commandSize -= size;
cmd->cursorOffset -= size;
cmd->screenOffset -= width;
cmd->endOffset -= width;
showOnScreen(cmd);
shellShowOnScreen(cmd);
}
}
void clearLineBefore(Command *cmd) {
void shellClearLineBefore(SShellCmd *cmd) {
assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset);
clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size);
shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE);
memmove(cmd->command, cmd->command + cmd->cursorOffset, cmd->commandSize - cmd->cursorOffset);
cmd->commandSize -= cmd->cursorOffset;
cmd->cursorOffset = 0;
cmd->screenOffset = 0;
cmd->endOffset = cmd->commandSize;
showOnScreen(cmd);
shellShowOnScreen(cmd);
}
void clearLineAfter(Command *cmd) {
void shellClearLineAfter(SShellCmd *cmd) {
assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset);
clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size);
shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE);
cmd->commandSize -= cmd->endOffset - cmd->cursorOffset;
cmd->endOffset = cmd->cursorOffset;
showOnScreen(cmd);
shellShowOnScreen(cmd);
}
void deleteChar(Command *cmd) {
void shellDeleteChar(SShellCmd *cmd) {
assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset);
if (cmd->cursorOffset < cmd->commandSize) {
clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size);
int size = 0;
int width = 0;
getNextCharSize(cmd->command, cmd->cursorOffset, &size, &width);
shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE);
int32_t size = 0;
int32_t width = 0;
shellGetNextCharSize(cmd->command, cmd->cursorOffset, &size, &width);
memmove(cmd->command + cmd->cursorOffset, cmd->command + cmd->cursorOffset + size,
cmd->commandSize - cmd->cursorOffset - size);
cmd->commandSize -= size;
cmd->endOffset -= width;
showOnScreen(cmd);
shellShowOnScreen(cmd);
}
}
void moveCursorLeft(Command *cmd) {
void shellMoveCursorLeft(SShellCmd *cmd) {
assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset);
if (cmd->cursorOffset > 0) {
clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size);
int size = 0;
int width = 0;
getPrevCharSize(cmd->command, cmd->cursorOffset, &size, &width);
shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE);
int32_t size = 0;
int32_t width = 0;
shellGetPrevCharSize(cmd->command, cmd->cursorOffset, &size, &width);
cmd->cursorOffset -= size;
cmd->screenOffset -= width;
showOnScreen(cmd);
shellShowOnScreen(cmd);
}
}
void moveCursorRight(Command *cmd) {
void shellMoveCursorRight(SShellCmd *cmd) {
assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset);
if (cmd->cursorOffset < cmd->commandSize) {
clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size);
int size = 0;
int width = 0;
getNextCharSize(cmd->command, cmd->cursorOffset, &size, &width);
shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE);
int32_t size = 0;
int32_t width = 0;
shellGetNextCharSize(cmd->command, cmd->cursorOffset, &size, &width);
cmd->cursorOffset += size;
cmd->screenOffset += width;
showOnScreen(cmd);
shellShowOnScreen(cmd);
}
}
void positionCursorHome(Command *cmd) {
void shellPositionCursorHome(SShellCmd *cmd) {
assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset);
if (cmd->cursorOffset > 0) {
clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size);
shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE);
cmd->cursorOffset = 0;
cmd->screenOffset = 0;
showOnScreen(cmd);
shellShowOnScreen(cmd);
}
}
void positionCursorEnd(Command *cmd) {
void shellPositionCursorEnd(SShellCmd *cmd) {
assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset);
if (cmd->cursorOffset < cmd->commandSize) {
clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size);
shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE);
cmd->cursorOffset = cmd->commandSize;
cmd->screenOffset = cmd->endOffset;
showOnScreen(cmd);
shellShowOnScreen(cmd);
}
}
void printChar(char c, int times) {
for (int i = 0; i < times; i++) {
void shellPrintChar(char c, int32_t times) {
for (int32_t i = 0; i < times; i++) {
fprintf(stdout, "%c", c);
}
fflush(stdout);
}
void positionCursor(int step, int direction) {
void shellPositionCursor(int32_t step, int32_t direction) {
if (step > 0) {
if (direction == LEFT) {
fprintf(stdout, "\033[%dD", step);
@ -211,32 +311,32 @@ void positionCursor(int step, int direction) {
}
}
void updateBuffer(Command *cmd) {
void shellUpdateBuffer(SShellCmd *cmd) {
assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset);
if (regex_match(cmd->buffer, "(\\s+$)|(^$)", REG_EXTENDED)) strcat(cmd->command, " ");
if (shellRegexMatch(cmd->buffer, "(\\s+$)|(^$)", REG_EXTENDED)) strcat(cmd->command, " ");
strcat(cmd->buffer, cmd->command);
cmd->bufferSize += cmd->commandSize;
memset(cmd->command, 0, MAX_COMMAND_SIZE);
memset(cmd->command, 0, SHELL_MAX_COMMAND_SIZE);
cmd->cursorOffset = 0;
cmd->screenOffset = 0;
cmd->commandSize = 0;
cmd->endOffset = 0;
showOnScreen(cmd);
shellShowOnScreen(cmd);
}
int isReadyGo(Command *cmd) {
int32_t shellIsReadyGo(SShellCmd *cmd) {
assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset);
char *total = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE);
memset(cmd->command + cmd->commandSize, 0, MAX_COMMAND_SIZE - cmd->commandSize);
char *total = (char *)taosMemoryCalloc(1, SHELL_MAX_COMMAND_SIZE);
memset(cmd->command + cmd->commandSize, 0, SHELL_MAX_COMMAND_SIZE - cmd->commandSize);
sprintf(total, "%s%s", cmd->buffer, cmd->command);
char *reg_str =
"(^.*;\\s*$)|(^\\s*$)|(^\\s*exit\\s*$)|(^\\s*q\\s*$)|(^\\s*quit\\s*$)|(^"
"\\s*clear\\s*$)";
if (regex_match(total, reg_str, REG_EXTENDED | REG_ICASE)) {
if (shellRegexMatch(total, reg_str, REG_EXTENDED | REG_ICASE)) {
taosMemoryFree(total);
return 1;
}
@ -245,28 +345,268 @@ int isReadyGo(Command *cmd) {
return 0;
}
void getMbSizeInfo(const char *str, int *size, int *width) {
TdWchar *wc = (TdWchar *)taosMemoryCalloc(sizeof(TdWchar), MAX_COMMAND_SIZE);
void shellGetMbSizeInfo(const char *str, int32_t *size, int32_t *width) {
TdWchar *wc = (TdWchar *)taosMemoryCalloc(sizeof(TdWchar), SHELL_MAX_COMMAND_SIZE);
*size = strlen(str);
taosMbsToWchars(wc, str, MAX_COMMAND_SIZE);
*width = taosWcharsWidth(wc, MAX_COMMAND_SIZE);
taosMbsToWchars(wc, str, SHELL_MAX_COMMAND_SIZE);
*width = taosWcharsWidth(wc, SHELL_MAX_COMMAND_SIZE);
taosMemoryFree(wc);
}
void resetCommand(Command *cmd, const char s[]) {
void shellResetCommand(SShellCmd *cmd, const char s[]) {
assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset);
clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size);
memset(cmd->buffer, 0, MAX_COMMAND_SIZE);
memset(cmd->command, 0, MAX_COMMAND_SIZE);
strncpy(cmd->command, s, MAX_COMMAND_SIZE);
int size = 0;
int width = 0;
getMbSizeInfo(s, &size, &width);
shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE);
memset(cmd->buffer, 0, SHELL_MAX_COMMAND_SIZE);
memset(cmd->command, 0, SHELL_MAX_COMMAND_SIZE);
strncpy(cmd->command, s, SHELL_MAX_COMMAND_SIZE);
int32_t size = 0;
int32_t width = 0;
shellGetMbSizeInfo(s, &size, &width);
cmd->bufferSize = 0;
cmd->commandSize = size;
cmd->cursorOffset = size;
cmd->screenOffset = width;
cmd->endOffset = width;
showOnScreen(cmd);
shellShowOnScreen(cmd);
}
void shellClearScreen(int32_t ecmd_pos, int32_t cursor_pos) {
struct winsize w;
if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) {
// fprintf(stderr, "No stream device, and use default value(col 120, row 30)\n");
w.ws_col = 120;
w.ws_row = 30;
}
int32_t cursor_x = cursor_pos / w.ws_col;
int32_t cursor_y = cursor_pos % w.ws_col;
int32_t command_x = ecmd_pos / w.ws_col;
shellPositionCursor(cursor_y, LEFT);
shellPositionCursor(command_x - cursor_x, DOWN);
fprintf(stdout, "\033[2K");
for (int32_t i = 0; i < command_x; i++) {
shellPositionCursor(1, UP);
fprintf(stdout, "\033[2K");
}
fflush(stdout);
}
void shellShowOnScreen(SShellCmd *cmd) {
struct winsize w;
if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) {
fprintf(stderr, "No stream device\n");
w.ws_col = 120;
w.ws_row = 30;
}
TdWchar wc;
int32_t size = 0;
// Print out the command.
char *total_string = taosMemoryMalloc(SHELL_MAX_COMMAND_SIZE);
memset(total_string, '\0', SHELL_MAX_COMMAND_SIZE);
if (strcmp(cmd->buffer, "") == 0) {
sprintf(total_string, "%s%s", shell.info.promptHeader, cmd->command);
} else {
sprintf(total_string, "%s%s", shell.info.promptContinue, cmd->command);
}
int32_t remain_column = w.ws_col;
for (char *str = total_string; size < cmd->commandSize + PSIZE;) {
int32_t ret = taosMbToWchar(&wc, str, MB_CUR_MAX);
if (ret < 0) break;
size += ret;
/* assert(size >= 0); */
int32_t width = taosWcharWidth(wc);
if (remain_column > width) {
printf("%lc", wc);
remain_column -= width;
} else {
if (remain_column == width) {
printf("%lc\n\r", wc);
remain_column = w.ws_col;
} else {
printf("\n\r%lc", wc);
remain_column = w.ws_col - width;
}
}
str = total_string + size;
}
taosMemoryFree(total_string);
// Position the cursor
int32_t cursor_pos = cmd->screenOffset + PSIZE;
int32_t ecmd_pos = cmd->endOffset + PSIZE;
int32_t cursor_x = cursor_pos / w.ws_col;
int32_t cursor_y = cursor_pos % w.ws_col;
// int32_t cursor_y = cursor % w.ws_col;
int32_t command_x = ecmd_pos / w.ws_col;
int32_t command_y = ecmd_pos % w.ws_col;
// int32_t command_y = (command.size() + PSIZE) % w.ws_col;
shellPositionCursor(command_y, LEFT);
shellPositionCursor(command_x, UP);
shellPositionCursor(cursor_x, DOWN);
shellPositionCursor(cursor_y, RIGHT);
fflush(stdout);
}
int32_t shellReadCommand(char *command) {
SShellHistory *pHistory = &shell.history;
SShellCmd cmd = {0};
uint32_t hist_counter = pHistory->hend;
char utf8_array[10] = "\0";
cmd.buffer = (char *)taosMemoryCalloc(1, SHELL_MAX_COMMAND_SIZE);
cmd.command = (char *)taosMemoryCalloc(1, SHELL_MAX_COMMAND_SIZE);
shellShowOnScreen(&cmd);
// Read input.
char c;
while (1) {
c = (char)getchar(); // getchar() return an 'int32_t' value
if (c == EOF) {
return c;
}
if (c < 0) { // For UTF-8
int32_t count = shellCountPrefixOnes(c);
utf8_array[0] = c;
for (int32_t k = 1; k < count; k++) {
c = (char)getchar();
utf8_array[k] = c;
}
shellInsertChar(&cmd, utf8_array, count);
} else if (c < '\033') {
// Ctrl keys. TODO: Implement ctrl combinations
switch (c) {
case 1: // ctrl A
shellPositionCursorHome(&cmd);
break;
case 3:
printf("\n");
shellResetCommand(&cmd, "");
kill(0, SIGINT);
break;
case 4: // EOF or Ctrl+D
printf("\n");
return -1;
case 5: // ctrl E
shellPositionCursorEnd(&cmd);
break;
case 8:
shellBackspaceChar(&cmd);
break;
case '\n':
case '\r':
printf("\n");
if (shellIsReadyGo(&cmd)) {
sprintf(command, "%s%s", cmd.buffer, cmd.command);
taosMemoryFreeClear(cmd.buffer);
taosMemoryFreeClear(cmd.command);
return 0;
} else {
shellUpdateBuffer(&cmd);
}
break;
case 11: // Ctrl + K;
shellClearLineAfter(&cmd);
break;
case 12: // Ctrl + L;
system("clear");
shellShowOnScreen(&cmd);
break;
case 21: // Ctrl + U;
shellClearLineBefore(&cmd);
break;
}
} else if (c == '\033') {
c = (char)getchar();
switch (c) {
case '[':
c = (char)getchar();
switch (c) {
case 'A': // Up arrow
if (hist_counter != pHistory->hstart) {
hist_counter = (hist_counter + SHELL_MAX_HISTORY_SIZE - 1) % SHELL_MAX_HISTORY_SIZE;
shellResetCommand(&cmd, (pHistory->hist[hist_counter] == NULL) ? "" : pHistory->hist[hist_counter]);
}
break;
case 'B': // Down arrow
if (hist_counter != pHistory->hend) {
int32_t next_hist = (hist_counter + 1) % SHELL_MAX_HISTORY_SIZE;
if (next_hist != pHistory->hend) {
shellResetCommand(&cmd, (pHistory->hist[next_hist] == NULL) ? "" : pHistory->hist[next_hist]);
} else {
shellResetCommand(&cmd, "");
}
hist_counter = next_hist;
}
break;
case 'C': // Right arrow
shellMoveCursorRight(&cmd);
break;
case 'D': // Left arrow
shellMoveCursorLeft(&cmd);
break;
case '1':
if ((c = (char)getchar()) == '~') {
// Home key
shellPositionCursorHome(&cmd);
}
break;
case '2':
if ((c = (char)getchar()) == '~') {
// Insert key
}
break;
case '3':
if ((c = (char)getchar()) == '~') {
// Delete key
shellDeleteChar(&cmd);
}
break;
case '4':
if ((c = (char)getchar()) == '~') {
// End key
shellPositionCursorEnd(&cmd);
}
break;
case '5':
if ((c = (char)getchar()) == '~') {
// Page up key
}
break;
case '6':
if ((c = (char)getchar()) == '~') {
// Page down key
}
break;
case 72:
// Home key
shellPositionCursorHome(&cmd);
break;
case 70:
// End key
shellPositionCursorEnd(&cmd);
break;
}
break;
}
} else if (c == 0x7f) {
// press delete key
shellBackspaceChar(&cmd);
} else {
shellInsertChar(&cmd, &c, 1);
}
}
return 0;
}
#endif

731
tools/shell/src/shellEngine.c
View File

File diff suppressed because it is too large Load Diff

728
tools/shell/src/shellMain.c
View File

@ -14,697 +14,53 @@
*/
#define __USE_XOPEN
#include "shellCommand.h"
#include "tglobal.h"
#include "tlog.h"
#include "shellInt.h"
#ifndef WINDOWS
#include <argp.h>
#include <termio.h>
#include <wordexp.h>
#endif
#define OPT_ABORT 1 /* abort */
int indicator = 1;
void insertChar(Command *cmd, char *c, int size);
void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, int32_t pkgNum, char *pkgType);
const char *argp_program_version = version;
const char *argp_program_bug_address = "<support@taosdata.com>";
static char doc[] = "";
static char args_doc[] = "";
TdThread pid;
static tsem_t cancelSem;
extern void taos_init();
#ifndef WINDOWS
static struct argp_option options[] = {
{"host", 'h', "HOST", 0, "TDengine server FQDN to connect. The default host is localhost."},
{"password", 'p', NULL, 0, "The password to use when connecting to the server."},
{"port", 'P', "PORT", 0, "The TCP/IP port number to use for the connection."},
{"user", 'u', "USER", 0, "The user name to use when connecting to the server."},
{"auth", 'A', "Auth", 0, "The auth string to use when connecting to the server."},
{"config-dir", 'c', "CONFIG_DIR", 0, "Configuration directory."},
{"dump-config",'C', NULL, 0, "Dump configuration."},
{"commands", 's', "COMMANDS", 0, "Commands to run without enter the shell."},
{"raw-time", 'r', NULL, 0, "Output time as uint64_t."},
{"file", 'f', "FILE", 0, "Script to run without enter the shell."},
{"directory", 'D', "DIRECTORY", 0, "Use multi-thread to import all SQL files in the directory separately."},
{"thread", 'T', "THREADNUM", 0, "Number of threads when using multi-thread to import data."},
{"check", 'k', "CHECK", 0, "Check tables."},
{"database", 'd', "DATABASE", 0, "Database to use when connecting to the server."},
{"timezone", 'z', "TIMEZONE", 0, "Time zone of the shell, default is local."},
{"status", 't', NULL, 0, "Check the service status."},
{"verbose", 'v', NULL, 0, "Check the details of the service status."},
{"netrole", 'n', "NETROLE", 0, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup|sync|speed|fqdn."},
{"pktlen", 'l', "PKTLEN", 0, "Packet length used for net test, default is 1000 bytes."},
{"pktnum", 'N', "PKTNUM", 0, "Packet numbers used for net test, default is 100."},
// Shuduo: 3.0 does not support UDP any more
// {"pkttype", 'S', "PKTTYPE", 0, "Packet type used for net test, default is TCP."},
{0}};
static error_t parse_opt(int key, char *arg, struct argp_state *state) {
/* Get the input argument from argp_parse, which we
know is a pointer to our arguments structure. */
SShellArguments *arguments = state->input;
wordexp_t full_path;
switch (key) {
case 'h':
arguments->host = arg;
break;
case 'p':
break;
case 'P':
if (arg) {
arguments->port = atoi(arg);
} else {
fprintf(stderr, "Invalid port\n");
return -1;
}
break;
case 'z':
arguments->timezone = arg;
break;
case 'u':
arguments->user = arg;
break;
case 'A':
arguments->auth = arg;
break;
case 'c':
if (wordexp(arg, &full_path, 0) != 0) {
fprintf(stderr, "Invalid path %s\n", arg);
return -1;
}
if (strlen(full_path.we_wordv[0]) >= TSDB_FILENAME_LEN) {
fprintf(stderr, "config file path: %s overflow max len %d\n", full_path.we_wordv[0], TSDB_FILENAME_LEN - 1);
wordfree(&full_path);
return -1;
}
tstrncpy(configDir, full_path.we_wordv[0], TSDB_FILENAME_LEN);
wordfree(&full_path);
break;
case 'C':
arguments->dump_config = true;
break;
case 's':
arguments->commands = arg;
break;
case 'r':
arguments->is_raw_time = true;
break;
case 'f':
if ((0 == strlen(arg)) || (wordexp(arg, &full_path, 0) != 0)) {
fprintf(stderr, "Invalid path %s\n", arg);
return -1;
}
tstrncpy(arguments->file, full_path.we_wordv[0], TSDB_FILENAME_LEN);
wordfree(&full_path);
break;
case 'D':
if (wordexp(arg, &full_path, 0) != 0) {
fprintf(stderr, "Invalid path %s\n", arg);
return -1;
}
tstrncpy(arguments->dir, full_path.we_wordv[0], TSDB_FILENAME_LEN);
wordfree(&full_path);
break;
case 'T':
if (arg) {
arguments->threadNum = atoi(arg);
} else {
fprintf(stderr, "Invalid number of threads\n");
return -1;
}
break;
case 'k':
arguments->check = atoi(arg);
break;
case 't':
arguments->status = true;
break;
case 'v':
arguments->verbose = true;
break;
case 'd':
arguments->database = arg;
break;
case 'n':
arguments->netTestRole = arg;
break;
case 'l':
if (arg) {
arguments->pktLen = atoi(arg);
} else {
fprintf(stderr, "Invalid packet length\n");
return -1;
}
break;
case 'N':
if (arg) {
arguments->pktNum = atoi(arg);
} else {
fprintf(stderr, "Invalid packet number\n");
return -1;
}
break;
case 'S':
arguments->pktType = arg;
break;
case OPT_ABORT:
arguments->abort = 1;
break;
default:
return ARGP_ERR_UNKNOWN;
}
return 0;
}
/* Our argp parser. */
static struct argp argp = {options, parse_opt, args_doc, doc};
char LINUXCLIENT_VERSION[] =
"Welcome to the TDengine shell from %s, Client Version:%s\n"
"Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n";
char g_password[SHELL_MAX_PASSWORD_LEN];
static void parse_args(int argc, char *argv[], SShellArguments *arguments) {
for (int i = 1; i < argc; i++) {
if ((strncmp(argv[i], "-p", 2) == 0) || (strncmp(argv[i], "--password", 10) == 0)) {
printf(LINUXCLIENT_VERSION, tsOsName, taos_get_client_info());
if ((strlen(argv[i]) == 2) || (strncmp(argv[i], "--password", 10) == 0)) {
printf("Enter password: ");
taosSetConsoleEcho(false);
if (scanf("%20s", g_password) > 1) {
fprintf(stderr, "password reading error\n");
}
taosSetConsoleEcho(true);
if (EOF == getchar()) {
fprintf(stderr, "getchar() return EOF\n");
}
} else {
tstrncpy(g_password, (char *)(argv[i] + 2), SHELL_MAX_PASSWORD_LEN);
strcpy(argv[i], "-p");
}
arguments->password = g_password;
arguments->is_use_passwd = true;
}
}
}
#endif
void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) {
#ifdef WINDOWS
#else
static char verType[32] = {0};
sprintf(verType, "version: %s\n", version);
argp_program_version = verType;
if (argc > 1) {
parse_args(argc, argv, arguments);
}
argp_parse(&argp, argc, argv, 0, 0, arguments);
if (arguments->abort) {
#ifndef _ALPINE
#if 0
error(10, 0, "ABORTED");
#endif
#else
abort();
#endif
}
#endif
}
int32_t shellReadCommand(TAOS *con, char *command) {
#ifdef WINDOWS
#else
unsigned hist_counter = history.hend;
char utf8_array[10] = "\0";
Command cmd;
memset(&cmd, 0, sizeof(cmd));
cmd.buffer = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE);
cmd.command = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE);
showOnScreen(&cmd);
// Read input.
char c;
while (1) {
c = (char)getchar(); // getchar() return an 'int' value
if (c == EOF) {
return c;
}
if (c < 0) { // For UTF-8
int count = countPrefixOnes(c);
utf8_array[0] = c;
for (int k = 1; k < count; k++) {
c = (char)getchar();
utf8_array[k] = c;
}
insertChar(&cmd, utf8_array, count);
} else if (c < '\033') {
// Ctrl keys. TODO: Implement ctrl combinations
switch (c) {
case 1: // ctrl A
positionCursorHome(&cmd);
break;
case 3:
printf("\n");
resetCommand(&cmd, "");
kill(0, SIGINT);
break;
case 4: // EOF or Ctrl+D
printf("\n");
taos_close(con);
// write the history
write_history();
exitShell();
break;
case 5: // ctrl E
positionCursorEnd(&cmd);
break;
case 8:
backspaceChar(&cmd);
break;
case '\n':
case '\r':
printf("\n");
if (isReadyGo(&cmd)) {
sprintf(command, "%s%s", cmd.buffer, cmd.command);
taosMemoryFreeClear(cmd.buffer);
taosMemoryFreeClear(cmd.command);
return 0;
} else {
updateBuffer(&cmd);
}
break;
case 11: // Ctrl + K;
clearLineAfter(&cmd);
break;
case 12: // Ctrl + L;
system("clear");
showOnScreen(&cmd);
break;
case 21: // Ctrl + U;
clearLineBefore(&cmd);
break;
}
} else if (c == '\033') {
c = (char)getchar();
switch (c) {
case '[':
c = (char)getchar();
switch (c) {
case 'A': // Up arrow
if (hist_counter != history.hstart) {
hist_counter = (hist_counter + MAX_HISTORY_SIZE - 1) % MAX_HISTORY_SIZE;
resetCommand(&cmd, (history.hist[hist_counter] == NULL) ? "" : history.hist[hist_counter]);
}
break;
case 'B': // Down arrow
if (hist_counter != history.hend) {
int next_hist = (hist_counter + 1) % MAX_HISTORY_SIZE;
if (next_hist != history.hend) {
resetCommand(&cmd, (history.hist[next_hist] == NULL) ? "" : history.hist[next_hist]);
} else {
resetCommand(&cmd, "");
}
hist_counter = next_hist;
}
break;
case 'C': // Right arrow
moveCursorRight(&cmd);
break;
case 'D': // Left arrow
moveCursorLeft(&cmd);
break;
case '1':
if ((c = (char)getchar()) == '~') {
// Home key
positionCursorHome(&cmd);
}
break;
case '2':
if ((c = (char)getchar()) == '~') {
// Insert key
}
break;
case '3':
if ((c = (char)getchar()) == '~') {
// Delete key
deleteChar(&cmd);
}
break;
case '4':
if ((c = (char)getchar()) == '~') {
// End key
positionCursorEnd(&cmd);
}
break;
case '5':
if ((c = (char)getchar()) == '~') {
// Page up key
}
break;
case '6':
if ((c = (char)getchar()) == '~') {
// Page down key
}
break;
case 72:
// Home key
positionCursorHome(&cmd);
break;
case 70:
// End key
positionCursorEnd(&cmd);
break;
}
break;
}
} else if (c == 0x7f) {
// press delete key
backspaceChar(&cmd);
} else {
insertChar(&cmd, &c, 1);
}
}
#endif
return 0;
}
void *shellLoopQuery(void *arg) {
if (indicator) {
getOldTerminalMode();
indicator = 0;
}
TAOS *con = (TAOS *)arg;
setThreadName("shellLoopQuery");
taosThreadCleanupPush(cleanup_handler, NULL);
char *command = taosMemoryMalloc(MAX_COMMAND_SIZE);
if (command == NULL) {
uError("failed to malloc command");
return NULL;
}
int32_t err = 0;
do {
// Read command from shell.
memset(command, 0, MAX_COMMAND_SIZE);
setTerminalMode();
err = shellReadCommand(con, command);
if (err) {
break;
}
resetTerminalMode();
} while (shellRunCommand(con, command) == 0);
taosMemoryFreeClear(command);
exitShell();
taosThreadCleanupPop(1);
return NULL;
}
void get_history_path(char *_history) { snprintf(_history, TSDB_FILENAME_LEN, "%s/%s", getenv("HOME"), HISTORY_FILE); }
void clearScreen(int ecmd_pos, int cursor_pos) {
#ifdef WINDOWS
#else
struct winsize w;
if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) {
// fprintf(stderr, "No stream device, and use default value(col 120, row 30)\n");
w.ws_col = 120;
w.ws_row = 30;
}
int cursor_x = cursor_pos / w.ws_col;
int cursor_y = cursor_pos % w.ws_col;
int command_x = ecmd_pos / w.ws_col;
positionCursor(cursor_y, LEFT);
positionCursor(command_x - cursor_x, DOWN);
fprintf(stdout, "\033[2K");
for (int i = 0; i < command_x; i++) {
positionCursor(1, UP);
fprintf(stdout, "\033[2K");
}
fflush(stdout);
#endif
}
void showOnScreen(Command *cmd) {
#ifdef WINDOWS
#else
struct winsize w;
if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) {
// fprintf(stderr, "No stream device\n");
w.ws_col = 120;
w.ws_row = 30;
}
TdWchar wc;
int size = 0;
// Print out the command.
char *total_string = taosMemoryMalloc(MAX_COMMAND_SIZE);
memset(total_string, '\0', MAX_COMMAND_SIZE);
if (strcmp(cmd->buffer, "") == 0) {
sprintf(total_string, "%s%s", PROMPT_HEADER, cmd->command);
} else {
sprintf(total_string, "%s%s", CONTINUE_PROMPT, cmd->command);
}
int remain_column = w.ws_col;
/* size = cmd->commandSize + prompt_size; */
for (char *str = total_string; size < cmd->commandSize + prompt_size;) {
int ret = taosMbToWchar(&wc, str, MB_CUR_MAX);
if (ret < 0) break;
size += ret;
/* assert(size >= 0); */
int width = taosWcharWidth(wc);
if (remain_column > width) {
printf("%lc", wc);
remain_column -= width;
} else {
if (remain_column == width) {
printf("%lc\n\r", wc);
remain_column = w.ws_col;
} else {
printf("\n\r%lc", wc);
remain_column = w.ws_col - width;
}
}
str = total_string + size;
}
taosMemoryFree(total_string);
/* for (int i = 0; i < size; i++){ */
/* char c = total_string[i]; */
/* if (k % w.ws_col == 0) { */
/* printf("%c\n\r", c); */
/* } */
/* else { */
/* printf("%c", c); */
/* } */
/* k += 1; */
/* } */
// Position the cursor
int cursor_pos = cmd->screenOffset + prompt_size;
int ecmd_pos = cmd->endOffset + prompt_size;
int cursor_x = cursor_pos / w.ws_col;
int cursor_y = cursor_pos % w.ws_col;
// int cursor_y = cursor % w.ws_col;
int command_x = ecmd_pos / w.ws_col;
int command_y = ecmd_pos % w.ws_col;
// int command_y = (command.size() + prompt_size) % w.ws_col;
positionCursor(command_y, LEFT);
positionCursor(command_x, UP);
positionCursor(cursor_x, DOWN);
positionCursor(cursor_y, RIGHT);
fflush(stdout);
#endif
}
void cleanup_handler(void *arg) { resetTerminalMode(); }
void exitShell() {
taos_cleanup();
exit(EXIT_SUCCESS);
}
void shellQueryInterruptHandler(int32_t signum, void *sigInfo, void *context) { tsem_post(&cancelSem); }
void *cancelHandler(void *arg) {
setThreadName("cancelHandler");
while (1) {
if (tsem_wait(&cancelSem) != 0) {
taosMsleep(10);
continue;
}
resetTerminalMode();
printf("\nReceive ctrl+c or other signal, quit shell.\n");
exitShell();
}
return NULL;
}
int checkVersion() {
if (sizeof(int8_t) != 1) {
printf("taos int8 size is %d(!= 1)", (int)sizeof(int8_t));
return 0;
}
if (sizeof(int16_t) != 2) {
printf("taos int16 size is %d(!= 2)", (int)sizeof(int16_t));
return 0;
}
if (sizeof(int32_t) != 4) {
printf("taos int32 size is %d(!= 4)", (int)sizeof(int32_t));
return 0;
}
if (sizeof(int64_t) != 8) {
printf("taos int64 size is %d(!= 8)", (int)sizeof(int64_t));
return 0;
}
return 1;
}
// Global configurations
SShellArguments args = {
.host = NULL,
.user = NULL,
.database = NULL,
.timezone = NULL,
.is_raw_time = false,
.is_use_passwd = false,
.dump_config = false,
.file = "\0",
.dir = "\0",
.threadNum = 5,
.commands = NULL,
.pktLen = 1000,
.pktNum = 100,
.pktType = "TCP",
.netTestRole = NULL,
#ifndef TD_WINDOWS
.password = NULL,
#endif
};
void shellDumpConfig() {
if (!args.dump_config) return;
SConfig *pCfg = taosGetCfg();
if (NULL == pCfg) {
printf("TDengine read global config failed!\n");
exit(EXIT_FAILURE);
}
cfgDumpCfg(pCfg, 0, 1);
exitShell();
}
void shellTestNetWork() {
if (args.netTestRole && args.netTestRole[0] != 0) {
taosNetTest(args.netTestRole, args.host, args.port, args.pktLen, args.pktNum, args.pktType);
exitShell();
}
}
void shellCheckServerStatus() {
if (!args.status && !args.verbose) return;
TSDB_SERVER_STATUS code;
do {
char details[1024] = {0};
code = taos_check_server_status(args.host, args.port, details, args.verbose ? 1024 : 0);
switch (code) {
case TSDB_SRV_STATUS_UNAVAILABLE:
printf("0: unavailable\n");
break;
case TSDB_SRV_STATUS_NETWORK_OK:
printf("1: network ok\n");
break;
case TSDB_SRV_STATUS_SERVICE_OK:
printf("2: service ok\n");
break;
case TSDB_SRV_STATUS_SERVICE_DEGRADED:
printf("3: service degraded\n");
break;
case TSDB_SRV_STATUS_EXTING:
printf("4: exiting\n");
break;
}
if (strlen(details) != 0) {
printf("%s\n\n", details);
}
if (code == TSDB_SRV_STATUS_NETWORK_OK && args.verbose) {
taosMsleep(1000);
} else {
break;
}
} while (1);
exitShell();
}
void shellExecute() {
TAOS *con = shellInit(&args);
if (con == NULL) {
exitShell();
}
if (tsem_init(&cancelSem, 0, 0) != 0) {
printf("failed to create cancel semphore\n");
exitShell();
}
TdThread spid;
taosThreadCreate(&spid, NULL, cancelHandler, NULL);
taosSetSignal(SIGTERM, shellQueryInterruptHandler);
taosSetSignal(SIGINT, shellQueryInterruptHandler);
taosSetSignal(SIGHUP, shellQueryInterruptHandler);
taosSetSignal(SIGABRT, shellQueryInterruptHandler);
shellGetGrantInfo(con);
while (1) {
taosThreadCreate(&pid, NULL, shellLoopQuery, con);
taosThreadJoin(pid, NULL);
}
}
SShellObj shell = {0};
int main(int argc, char *argv[]) {
if (!checkVersion()) exitShell();
if (shellCheckIntSize() != 0) {
return -1;
}
shellParseArgument(argc, argv, &args);
if (shellParseArgs(argc, argv) != 0) {
return -1;
}
if (shell.args.is_version) {
shellPrintVersion();
return 0;
}
if (shell.args.is_gen_auth) {
shellGenerateAuth();
return 0;
}
if (shell.args.is_help) {
shellPrintHelp();
return 0;
}
taos_init();
shellDumpConfig();
shellCheckServerStatus();
shellTestNetWork();
shellExecute();
return 0;
if (shell.args.is_dump_config) {
shellDumpConfig();
taos_cleanup();
return 0;
}
if (shell.args.is_startup || shell.args.is_check) {
shellCheckServerStatus();
taos_cleanup();
return 0;
}
if (shell.args.netrole != NULL) {
shellTestNetWork();
taos_cleanup();
return 0;
}
return shellExecute();
}

145
tools/shell/src/shellNettest.c Normal file
View File

@ -0,0 +1,145 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _GNU_SOURCE
#include "shellInt.h"
static void shellWorkAsClient() {
SShellArgs *pArgs = &shell.args;
SRpcInit rpcInit = {0};
SEpSet epSet = {.inUse = 0, .numOfEps = 1};
SRpcMsg rpcRsp = {0};
void *clientRpc = NULL;
char pass[TSDB_PASSWORD_LEN + 1] = {0};
taosEncryptPass_c((uint8_t *)("_pwd"), strlen("_pwd"), pass);
rpcInit.label = "CHK";
rpcInit.numOfThreads = 1;
rpcInit.sessions = 16;
rpcInit.connType = TAOS_CONN_CLIENT;
rpcInit.idleTime = tsShellActivityTimer * 1000;
rpcInit.user = "_dnd";
rpcInit.ckey = "_key";
rpcInit.spi = 1;
rpcInit.secret = pass;
clientRpc = rpcOpen(&rpcInit);
if (clientRpc == NULL) {
printf("failed to init net test client since %s\n", terrstr());
goto _OVER;
}
if (pArgs->host == NULL) {
pArgs->host = tsFirst;
}
char fqdn[TSDB_FQDN_LEN] = {0};
tstrncpy(fqdn, pArgs->host, TSDB_FQDN_LEN);
strtok(fqdn, ":");
if (pArgs->port == 0) {
pArgs->port = tsServerPort;
}
printf("network test client is initialized, the server is %s:%u\n", fqdn, pArgs->port);
tstrncpy(epSet.eps[0].fqdn, fqdn, TSDB_FQDN_LEN);
epSet.eps[0].port = (uint16_t)pArgs->port;
int32_t totalSucc = 0;
uint64_t startTime = taosGetTimestampUs();
for (int32_t i = 0; i < pArgs->pktNum; ++i) {
SRpcMsg rpcMsg = {.ahandle = (void *)0x9525, .msgType = TDMT_DND_NET_TEST};
rpcMsg.pCont = rpcMallocCont(pArgs->pktLen);
rpcMsg.contLen = pArgs->pktLen;
printf("request is sent, size:%d\n", rpcMsg.contLen);
rpcSendRecv(clientRpc, &epSet, &rpcMsg, &rpcRsp);
if (rpcRsp.code == 0 && rpcRsp.contLen == rpcMsg.contLen) {
printf("response is received, size:%d\n", rpcMsg.contLen);
if (rpcRsp.code == 0) totalSucc++;
} else {
printf("response not received since %s\n", tstrerror(rpcRsp.code));
}
rpcFreeCont(rpcRsp.pCont);
rpcRsp.pCont = NULL;
}
uint64_t endTime = taosGetTimestampUs();
uint64_t elT = endTime - startTime;
printf("\ntotal succ:%5d/%d\tcost:%8.2lf ms\tspeed:%8.2lf MB/s\n", totalSucc, pArgs->pktNum, elT / 1000.0,
pArgs->pktLen / (elT / 1000000.0) / 1024.0 / 1024.0 * totalSucc);
_OVER:
if (clientRpc != NULL) {
rpcClose(clientRpc);
}
if (rpcRsp.pCont != NULL) {
rpcFreeCont(rpcRsp.pCont);
}
}
static void shellProcessMsg(void *p, SRpcMsg *pRpc, SEpSet *pEpSet) {
printf("request is received, size:%d\n", pRpc->contLen);
fflush(stdout);
SRpcMsg rsp = {.handle = pRpc->handle, .refId = pRpc->refId, .ahandle = pRpc->ahandle, .code = 0};
rsp.pCont = rpcMallocCont(pRpc->contLen);
if (rsp.pCont == NULL) {
rsp.code = TSDB_CODE_OUT_OF_MEMORY;
} else {
rsp.contLen = pRpc->contLen;
}
rpcSendResponse(&rsp);
}
void shellNettestHandler(int32_t signum, void *sigInfo, void *context) { shellExit(); }
static void shellWorkAsServer() {
SShellArgs *pArgs = &shell.args;
if (pArgs->port == 0) {
pArgs->port = tsServerPort;
}
SRpcInit rpcInit = {0};
rpcInit.localPort = pArgs->port;
rpcInit.label = "CHK";
rpcInit.numOfThreads = tsNumOfRpcThreads;
rpcInit.cfp = (RpcCfp)shellProcessMsg;
rpcInit.sessions = 10;
rpcInit.connType = TAOS_CONN_SERVER;
rpcInit.idleTime = tsShellActivityTimer * 1000;
void *serverRpc = rpcOpen(&rpcInit);
if (serverRpc == NULL) {
printf("failed to init net test server since %s", terrstr());
} else {
printf("network test server is initialized, port:%u\n", pArgs->port);
taosSetSignal(SIGTERM, shellNettestHandler);
while (1) taosMsleep(10);
}
}
void shellTestNetWork() {
if (strcmp(shell.args.netrole, "client") == 0) {
shellWorkAsClient();
}
if (strcmp(shell.args.netrole, "server") == 0) {
shellWorkAsServer();
}
}

132
tools/shell/src/shellUtil.c Normal file
View File

@ -0,0 +1,132 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _BSD_SOURCE
#define _GNU_SOURCE
#define _XOPEN_SOURCE
#define _DEFAULT_SOURCE
#include "shellInt.h"
bool shellRegexMatch(const char *s, const char *reg, int32_t cflags) {
regex_t regex = {0};
char msgbuf[100] = {0};
/* Compile regular expression */
if (regcomp(&regex, reg, cflags) != 0) {
fprintf(stderr, "Fail to compile regex");
shellExit();
}
/* Execute regular expression */
int32_t reti = regexec(&regex, s, 0, NULL, 0);
if (!reti) {
regfree(&regex);
return true;
} else if (reti == REG_NOMATCH) {
regfree(&regex);
return false;
} else {
regerror(reti, &regex, msgbuf, sizeof(msgbuf));
fprintf(stderr, "Regex match failed: %s\n", msgbuf);
regfree(&regex);
shellExit();
}
return false;
}
int32_t shellCheckIntSize() {
if (sizeof(int8_t) != 1) {
printf("taos int8 size is %d(!= 1)", (int)sizeof(int8_t));
return -1;
}
if (sizeof(int16_t) != 2) {
printf("taos int16 size is %d(!= 2)", (int)sizeof(int16_t));
return -1;
}
if (sizeof(int32_t) != 4) {
printf("taos int32 size is %d(!= 4)", (int)sizeof(int32_t));
return -1;
}
if (sizeof(int64_t) != 8) {
printf("taos int64 size is %d(!= 8)", (int)sizeof(int64_t));
return -1;
}
return 0;
}
void shellPrintVersion() { printf("version: %s\n", version); }
void shellGenerateAuth() {
char secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0};
taosEncryptPass_c((uint8_t *)shell.args.password, strlen(shell.args.password), secretEncrypt);
printf("%s\n", secretEncrypt);
fflush(stdout);
}
void shellDumpConfig() {
SConfig *pCfg = taosGetCfg();
if (pCfg == NULL) {
printf("TDengine read global config failed!\n");
} else {
cfgDumpCfg(pCfg, 1, true);
}
fflush(stdout);
}
void shellCheckServerStatus() {
TSDB_SERVER_STATUS code;
do {
char details[1024] = {0};
code = taos_check_server_status(shell.args.host, shell.args.port, details, 1024);
switch (code) {
case TSDB_SRV_STATUS_UNAVAILABLE:
printf("0: unavailable\n");
break;
case TSDB_SRV_STATUS_NETWORK_OK:
printf("1: network ok\n");
break;
case TSDB_SRV_STATUS_SERVICE_OK:
printf("2: service ok\n");
break;
case TSDB_SRV_STATUS_SERVICE_DEGRADED:
printf("3: service degraded\n");
break;
case TSDB_SRV_STATUS_EXTING:
printf("4: exiting\n");
break;
}
if (strlen(details) != 0) {
printf("%s\n\n", details);
}
fflush(stdout);
if (code == TSDB_SRV_STATUS_NETWORK_OK && shell.args.is_startup) {
taosMsleep(1000);
} else {
break;
}
} while (1);
}
void shellExit() {
if (shell.conn != NULL) {
taos_close(shell.conn);
shell.conn = NULL;
}
taos_cleanup();
exit(EXIT_FAILURE);
}

514
tools/shell/src/tnettest.c
View File

@ -1,514 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#define ALLOW_FORBID_FUNC
#include "os.h"
#include "taosdef.h"
#include "tmsg.h"
#include "taoserror.h"
#include "tlog.h"
#include "tglobal.h"
#include "trpc.h"
#include "rpcHead.h"
#include "tchecksum.h"
#include "syncMsg.h"
#include "osSocket.h"
#define MAX_PKG_LEN (64 * 1000)
#define MAX_SPEED_PKG_LEN (1024 * 1024 * 1024)
#define MIN_SPEED_PKG_LEN 1024
#define MAX_SPEED_PKG_NUM 10000
#define MIN_SPEED_PKG_NUM 1
#define BUFFER_SIZE (MAX_PKG_LEN + 1024)
extern int tsRpcMaxUdpSize;
typedef struct {
char * hostFqdn;
uint32_t hostIp;
int32_t port;
int32_t pktLen;
} STestInfo;
static void *taosNetBindUdpPort(void *sarg) {
STestInfo *pinfo = (STestInfo *)sarg;
int32_t port = pinfo->port;
SOCKET serverSocket;
char buffer[BUFFER_SIZE];
int32_t iDataNum;
socklen_t sin_size;
int32_t bufSize = 1024000;
struct sockaddr_in server_addr;
struct sockaddr_in clientAddr;
setThreadName("netBindUdpPort");
if ((serverSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) {
uError("failed to create UDP socket since %s", strerror(errno));
return NULL;
}
bzero(&server_addr, sizeof(server_addr));
server_addr.sin_family = AF_INET;
server_addr.sin_port = htons(port);
server_addr.sin_addr.s_addr = htonl(INADDR_ANY);
if (bind(serverSocket, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) {
uError("failed to bind UDP port:%d since %s", port, strerror(errno));
return NULL;
}
TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket));
if (pSocket == NULL) {
taosCloseSocketNoCheck1(serverSocket);
return NULL;
}
pSocket->fd = serverSocket;
pSocket->refId = 0;
if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) {
uError("failed to set the send buffer size for UDP socket\n");
taosCloseSocket(&pSocket);
return NULL;
}
if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) {
uError("failed to set the receive buffer size for UDP socket\n");
taosCloseSocket(&pSocket);
return NULL;
}
uInfo("UDP server at port:%d is listening", port);
while (1) {
memset(buffer, 0, BUFFER_SIZE);
sin_size = sizeof(*(struct sockaddr *)&server_addr);
iDataNum = recvfrom(serverSocket, buffer, BUFFER_SIZE, 0, (struct sockaddr *)&clientAddr, &sin_size);
if (iDataNum < 0) {
uDebug("failed to perform recvfrom func at %d since %s", port, strerror(errno));
continue;
}
uInfo("UDP: recv:%d bytes from %s at %d", iDataNum, taosInetNtoa(clientAddr.sin_addr), port);
if (iDataNum > 0) {
iDataNum = taosSendto(pSocket, buffer, iDataNum, 0, (struct sockaddr *)&clientAddr, (int32_t)sin_size);
}
uInfo("UDP: send:%d bytes to %s at %d", iDataNum, taosInetNtoa(clientAddr.sin_addr), port);
}
taosCloseSocket(&pSocket);
return NULL;
}
static void *taosNetBindTcpPort(void *sarg) {
struct sockaddr_in server_addr;
struct sockaddr_in clientAddr;
STestInfo *pinfo = sarg;
int32_t port = pinfo->port;
SOCKET serverSocket;
int32_t addr_len = sizeof(clientAddr);
SOCKET client;
char buffer[BUFFER_SIZE];
setThreadName("netBindTcpPort");
if ((serverSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) {
uError("failed to create TCP socket since %s", strerror(errno));
return NULL;
}
bzero(&server_addr, sizeof(server_addr));
server_addr.sin_family = AF_INET;
server_addr.sin_port = htons(port);
server_addr.sin_addr.s_addr = htonl(INADDR_ANY);
int32_t reuse = 1;
TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket));
if (pSocket == NULL) {
taosCloseSocketNoCheck1(serverSocket);
return NULL;
}
pSocket->fd = serverSocket;
pSocket->refId = 0;
if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) {
uError("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno));
taosCloseSocket(&pSocket);
return NULL;
}
if (bind(serverSocket, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) {
uError("failed to bind TCP port:%d since %s", port, strerror(errno));
taosCloseSocket(&pSocket);
return NULL;
}
if (taosKeepTcpAlive(pSocket) < 0) {
uError("failed to set tcp server keep-alive option since %s", strerror(errno));
taosCloseSocket(&pSocket);
return NULL;
}
if (listen(serverSocket, 10) < 0) {
uError("failed to listen TCP port:%d since %s", port, strerror(errno));
taosCloseSocket(&pSocket);
return NULL;
}
uInfo("TCP server at port:%d is listening", port);
while (1) {
client = accept(serverSocket, (struct sockaddr *)&clientAddr, (socklen_t *)&addr_len);
if (client < 0) {
uDebug("TCP: failed to accept at port:%d since %s", port, strerror(errno));
continue;
}
int32_t ret = taosReadMsg(pSocket, buffer, pinfo->pktLen);
if (ret < 0 || ret != pinfo->pktLen) {
uError("TCP: failed to read %d bytes at port:%d since %s", pinfo->pktLen, port, strerror(errno));
taosCloseSocket(&pSocket);
return NULL;
}
uInfo("TCP: read:%d bytes from %s at %d", pinfo->pktLen, taosInetNtoa(clientAddr.sin_addr), port);
ret = taosWriteMsg(pSocket, buffer, pinfo->pktLen);
if (ret < 0) {
uError("TCP: failed to write %d bytes at %d since %s", pinfo->pktLen, port, strerror(errno));
taosCloseSocket(&pSocket);
return NULL;
}
uInfo("TCP: write:%d bytes to %s at %d", pinfo->pktLen, taosInetNtoa(clientAddr.sin_addr), port);
}
taosCloseSocket(&pSocket);
return NULL;
}
static int32_t taosNetCheckTcpPort(STestInfo *info) {
SOCKET clientSocket;
char buffer[BUFFER_SIZE] = {0};
if ((clientSocket = socket(AF_INET, SOCK_STREAM, 0)) < 0) {
uError("failed to create TCP client socket since %s", strerror(errno));
return -1;
}
int32_t reuse = 1;
TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket));
if (pSocket == NULL) {
taosCloseSocketNoCheck1(clientSocket);
return -1;
}
pSocket->fd = clientSocket;
pSocket->refId = 0;
if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) {
uError("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno));
taosCloseSocket(&pSocket);
return -1;
}
struct sockaddr_in serverAddr;
memset((char *)&serverAddr, 0, sizeof(serverAddr));
serverAddr.sin_family = AF_INET;
serverAddr.sin_port = (uint16_t)htons((uint16_t)info->port);
serverAddr.sin_addr.s_addr = info->hostIp;
if (connect(clientSocket, (struct sockaddr *)&serverAddr, sizeof(serverAddr)) < 0) {
uError("TCP: failed to connect port %s:%d since %s", taosIpStr(info->hostIp), info->port, strerror(errno));
taosCloseSocket(&pSocket);
return -1;
}
taosKeepTcpAlive(pSocket);
sprintf(buffer, "client send TCP pkg to %s:%d, content: 1122334455", taosIpStr(info->hostIp), info->port);
sprintf(buffer + info->pktLen - 16, "1122334455667788");
int32_t ret = taosWriteMsg(pSocket, buffer, info->pktLen);
if (ret < 0) {
uError("TCP: failed to write msg to %s:%d since %s", taosIpStr(info->hostIp), info->port, strerror(errno));
taosCloseSocket(&pSocket);
return -1;
}
ret = taosReadMsg(pSocket, buffer, info->pktLen);
if (ret < 0) {
uError("TCP: failed to read msg from %s:%d since %s", taosIpStr(info->hostIp), info->port, strerror(errno));
taosCloseSocket(&pSocket);
return -1;
}
taosCloseSocket(&pSocket);
return 0;
}
static int32_t taosNetCheckUdpPort(STestInfo *info) {
SOCKET clientSocket;
char buffer[BUFFER_SIZE] = {0};
int32_t iDataNum = 0;
int32_t bufSize = 1024000;
struct sockaddr_in serverAddr;
if ((clientSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) {
uError("failed to create udp client socket since %s", strerror(errno));
return -1;
}
TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket));
if (pSocket == NULL) {
taosCloseSocketNoCheck1(clientSocket);
return -1;
}
pSocket->fd = clientSocket;
pSocket->refId = 0;
if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) {
uError("failed to set the send buffer size for UDP socket\n");
taosCloseSocket(&pSocket);
return -1;
}
if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) {
uError("failed to set the receive buffer size for UDP socket\n");
taosCloseSocket(&pSocket);
return -1;
}
serverAddr.sin_family = AF_INET;
serverAddr.sin_port = htons(info->port);
serverAddr.sin_addr.s_addr = info->hostIp;
struct in_addr ipStr;
memcpy(&ipStr, &info->hostIp, 4);
sprintf(buffer, "client send udp pkg to %s:%d, content: 1122334455", taosInetNtoa(ipStr), info->port);
sprintf(buffer + info->pktLen - 16, "1122334455667788");
socklen_t sin_size = sizeof(*(struct sockaddr *)&serverAddr);
iDataNum = taosSendto(pSocket, buffer, info->pktLen, 0, (struct sockaddr *)&serverAddr, (int32_t)sin_size);
if (iDataNum < 0 || iDataNum != info->pktLen) {
uError("UDP: failed to perform sendto func since %s", strerror(errno));
taosCloseSocket(&pSocket);
return -1;
}
memset(buffer, 0, BUFFER_SIZE);
sin_size = sizeof(*(struct sockaddr *)&serverAddr);
iDataNum = recvfrom(clientSocket, buffer, BUFFER_SIZE, 0, (struct sockaddr *)&serverAddr, &sin_size);
if (iDataNum < 0 || iDataNum != info->pktLen) {
uError("UDP: received ack:%d bytes(expect:%d) from port:%d since %s", iDataNum, info->pktLen, info->port, strerror(errno));
taosCloseSocket(&pSocket);
return -1;
}
taosCloseSocket(&pSocket);
return 0;
}
static void taosNetCheckPort(uint32_t hostIp, int32_t startPort, int32_t endPort, int32_t pktLen) {
int32_t ret;
STestInfo info;
memset(&info, 0, sizeof(STestInfo));
info.hostIp = hostIp;
info.pktLen = pktLen;
for (int32_t port = startPort; port <= endPort; port++) {
info.port = port;
ret = taosNetCheckTcpPort(&info);
if (ret != 0) {
printf("failed to test TCP port:%d\n", port);
} else {
printf("successed to test TCP port:%d\n", port);
}
ret = taosNetCheckUdpPort(&info);
if (ret != 0) {
printf("failed to test UDP port:%d\n", port);
} else {
printf("successed to test UDP port:%d\n", port);
}
}
}
static void taosNetTestClient(char *host, int32_t startPort, int32_t pkgLen) {
uInfo("work as client, host:%s Port:%d pkgLen:%d\n", host, startPort, pkgLen);
uint32_t serverIp = taosGetIpv4FromFqdn(host);
if (serverIp == 0xFFFFFFFF) {
uError("failed to resolve fqdn:%s", host);
exit(-1);
}
uInfo("server ip:%s is resolved from host:%s", taosIpStr(serverIp), host);
taosNetCheckPort(serverIp, startPort, startPort, pkgLen);
}
static void taosNetTestServer(char *host, int32_t startPort, int32_t pkgLen) {
uInfo("work as server, host:%s Port:%d pkgLen:%d\n", host, startPort, pkgLen);
int32_t port = startPort;
int32_t num = 1;
if (num < 0) num = 1;
TdThread *pids = taosMemoryMalloc(2 * num * sizeof(TdThread));
STestInfo *tinfos = taosMemoryMalloc(num * sizeof(STestInfo));
STestInfo *uinfos = taosMemoryMalloc(num * sizeof(STestInfo));
for (int32_t i = 0; i < num; i++) {
STestInfo *tcpInfo = tinfos + i;
tcpInfo->port = port + i;
tcpInfo->pktLen = pkgLen;
if (taosThreadCreate(pids + i, NULL, taosNetBindTcpPort, tcpInfo) != 0) {
uInfo("failed to create TCP test thread, %s:%d", tcpInfo->hostFqdn, tcpInfo->port);
exit(-1);
}
STestInfo *udpInfo = uinfos + i;
udpInfo->port = port + i;
tcpInfo->pktLen = pkgLen;
if (taosThreadCreate(pids + num + i, NULL, taosNetBindUdpPort, udpInfo) != 0) {
uInfo("failed to create UDP test thread, %s:%d", tcpInfo->hostFqdn, tcpInfo->port);
exit(-1);
}
}
for (int32_t i = 0; i < num; i++) {
taosThreadJoin(pids[i], NULL);
taosThreadJoin(pids[(num + i)], NULL);
}
}
static void taosNetCheckSpeed(char *host, int32_t port, int32_t pkgLen,
int32_t pkgNum, char *pkgType) {
#if 0
// record config
int32_t compressTmp = tsCompressMsgSize;
int32_t maxUdpSize = tsRpcMaxUdpSize;
int32_t forceTcp = tsRpcForceTcp;
if (0 == strcmp("tcp", pkgType)){
tsRpcForceTcp = 1;
tsRpcMaxUdpSize = 0; // force tcp
} else {
tsRpcForceTcp = 0;
tsRpcMaxUdpSize = INT_MAX;
}
tsCompressMsgSize = -1;
SEpSet epSet;
SRpcMsg reqMsg;
SRpcMsg rspMsg;
void * pRpcConn;
char secretEncrypt[32] = {0};
char spi = 0;
pRpcConn = taosNetInitRpc(secretEncrypt, spi);
if (NULL == pRpcConn) {
uError("failed to init client rpc");
return;
}
printf("check net spend, host:%s port:%d pkgLen:%d pkgNum:%d pkgType:%s\n\n", host, port, pkgLen, pkgNum, pkgType);
int32_t totalSucc = 0;
uint64_t startT = taosGetTimestampUs();
for (int32_t i = 1; i <= pkgNum; i++) {
uint64_t startTime = taosGetTimestampUs();
memset(&epSet, 0, sizeof(SEpSet));
strcpy(epSet.eps[0].fqdn, host);
epSet.eps[0].port = port;
epSet.numOfEps = 1;
reqMsg.msgType = TDMT_DND_NETWORK_TEST;
reqMsg.pCont = rpcMallocCont(pkgLen);
reqMsg.contLen = pkgLen;
reqMsg.code = 0;
reqMsg.handle = NULL; // rpc handle returned to app
reqMsg.ahandle = NULL; // app handle set by client
strcpy(reqMsg.pCont, "nettest speed");
rpcSendRecv(pRpcConn, &epSet, &reqMsg, &rspMsg);
int code = 0;
if ((rspMsg.code != 0) || (rspMsg.msgType != TDMT_DND_NETWORK_TEST + 1)) {
uError("ret code 0x%x %s", rspMsg.code, tstrerror(rspMsg.code));
code = -1;
}else{
totalSucc ++;
}
rpcFreeCont(rspMsg.pCont);
uint64_t endTime = taosGetTimestampUs();
uint64_t el = endTime - startTime;
printf("progress:%5d/%d\tstatus:%d\tcost:%8.2lf ms\tspeed:%8.2lf MB/s\n", i, pkgNum, code, el/1000.0, pkgLen/(el/1000000.0)/1024.0/1024.0);
}
int64_t endT = taosGetTimestampUs();
uint64_t elT = endT - startT;
printf("\ntotal succ:%5d/%d\tcost:%8.2lf ms\tspeed:%8.2lf MB/s\n", totalSucc, pkgNum, elT/1000.0, pkgLen/(elT/1000000.0)/1024.0/1024.0*totalSucc);
rpcClose(pRpcConn);
// return config
tsCompressMsgSize = compressTmp;
tsRpcMaxUdpSize = maxUdpSize;
tsRpcForceTcp = forceTcp;
return;
#endif
}
void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, int32_t pkgNum, char *pkgType) {
tsLogEmbedded = 1;
if (host == NULL) host = tsLocalFqdn;
if (port == 0) port = tsServerPort;
if (0 == strcmp("speed", role)) {
if (pkgLen <= MIN_SPEED_PKG_LEN) pkgLen = MIN_SPEED_PKG_LEN;
if (pkgLen > MAX_SPEED_PKG_LEN) pkgLen = MAX_SPEED_PKG_LEN;
if (pkgNum <= MIN_SPEED_PKG_NUM) pkgNum = MIN_SPEED_PKG_NUM;
if (pkgNum > MAX_SPEED_PKG_NUM) pkgNum = MAX_SPEED_PKG_NUM;
} else {
if (pkgLen <= 10) pkgLen = 1000;
if (pkgLen > MAX_PKG_LEN) pkgLen = MAX_PKG_LEN;
}
if (0 == strcmp("client", role)) {
taosNetTestClient(host, port, pkgLen);
} else if (0 == strcmp("server", role)) {
taosNetTestServer(host, port, pkgLen);
} else if (0 == strcmp("speed", role)) {
tsLogEmbedded = 0;
char type[10] = {0};
taosNetCheckSpeed(host, port, pkgLen, pkgNum, strtolower(type, pkgType));
} else {
TASSERT(1);
}
tsLogEmbedded = 0;
}