xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

conflict resolve

This commit is contained in:
Cary Xu 2021-07-17 09:46:30 +08:00
parent 3ecbc86054 67e877ce0e
commit f06ec62786
79 changed files with 7719 additions and 2171 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
cmake/define.inc
View File

@ -83,6 +83,8 @@ IF (TD_ARM_64)
ADD_DEFINITIONS(-DUSE_LIBICONV)
MESSAGE(STATUS "arm64 is defined")
SET(COMMON_FLAGS "-Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lua/src)
ENDIF ()
IF (TD_ARM_32)
@ -91,6 +93,8 @@ IF (TD_ARM_32)
ADD_DEFINITIONS(-DUSE_LIBICONV)
MESSAGE(STATUS "arm32 is defined")
SET(COMMON_FLAGS "-Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE -Wno-pointer-to-int-cast -Wno-int-to-pointer-cast -Wno-incompatible-pointer-types ")
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lua/src)
ENDIF ()
IF (TD_MIPS_64)
@ -143,6 +147,7 @@ IF (TD_LINUX)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lua/src)
ENDIF ()
IF (TD_DARWIN_64)
@ -164,6 +169,7 @@ IF (TD_DARWIN_64)
SET(RELEASE_FLAGS "-Og")
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lua/src)
ENDIF ()
IF (TD_WINDOWS)
@ -194,6 +200,7 @@ IF (TD_WINDOWS)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/regex)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/wepoll/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/MsvcLibX/include)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lua/src)
ENDIF ()
IF (TD_WINDOWS_64)

12
src/client/CMakeLists.txt
View File

@ -21,7 +21,7 @@ IF (TD_LINUX)
IF (TD_LINUX_64)
TARGET_LINK_LIBRARIES(taos lua)
ENDIF ()
SET_TARGET_PROPERTIES(taos PROPERTIES CLEAN_DIRECT_OUTPUT 1)
#set version of .so
@ -37,13 +37,13 @@ ELSEIF (TD_DARWIN)
# set the static lib name
ADD_LIBRARY(taos_static STATIC ${SRC})
TARGET_LINK_LIBRARIES(taos_static common query trpc tutil pthread m)
TARGET_LINK_LIBRARIES(taos_static common query trpc tutil pthread m lua)
SET_TARGET_PROPERTIES(taos_static PROPERTIES OUTPUT_NAME "taos_static")
SET_TARGET_PROPERTIES(taos_static PROPERTIES CLEAN_DIRECT_OUTPUT 1)
# generate dynamic library (*.dylib)
ADD_LIBRARY(taos SHARED ${SRC})
TARGET_LINK_LIBRARIES(taos common query trpc tutil pthread m)
TARGET_LINK_LIBRARIES(taos common query trpc tutil pthread m lua)
SET_TARGET_PROPERTIES(taos PROPERTIES CLEAN_DIRECT_OUTPUT 1)
#set version of .dylib
@ -68,19 +68,19 @@ ELSEIF (TD_WINDOWS)
IF (NOT TD_GODLL)
SET_TARGET_PROPERTIES(taos PROPERTIES LINK_FLAGS /DEF:${TD_COMMUNITY_DIR}/src/client/src/taos.def)
ENDIF ()
TARGET_LINK_LIBRARIES(taos trpc tutil query)
TARGET_LINK_LIBRARIES(taos trpc tutil query lua)
ELSEIF (TD_DARWIN)
SET(CMAKE_MACOSX_RPATH 1)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/linux)
ADD_LIBRARY(taos_static STATIC ${SRC})
TARGET_LINK_LIBRARIES(taos_static query trpc tutil pthread m)
TARGET_LINK_LIBRARIES(taos_static query trpc tutil pthread m lua)
SET_TARGET_PROPERTIES(taos_static PROPERTIES OUTPUT_NAME "taos_static")
# generate dynamic library (*.dylib)
ADD_LIBRARY(taos SHARED ${SRC})
TARGET_LINK_LIBRARIES(taos query trpc tutil pthread m)
TARGET_LINK_LIBRARIES(taos query trpc tutil pthread m lua)
SET_TARGET_PROPERTIES(taos PROPERTIES CLEAN_DIRECT_OUTPUT 1)

4
src/client/inc/tscUtil.h
View File

@ -117,6 +117,7 @@ SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint
uint32_t offset);
void* tscDestroyBlockArrayList(SArray* pDataBlockList);
void* tscDestroyUdfArrayList(SArray* pUdfList);
void* tscDestroyBlockHashTable(SHashObj* pBlockHashTable, bool removeMeta);
int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock);
@ -271,6 +272,7 @@ void tscVgroupTableCopy(SVgroupTableInfo* info, SVgroupTableInfo* pInfo);
int tscGetSTableVgroupInfo(SSqlObj* pSql, SQueryInfo* pQueryInfo);
int tscGetTableMeta(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo);
int tscGetTableMetaEx(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, bool createIfNotExists);
int32_t tscGetUdfFromNode(SSqlObj *pSql, SQueryInfo* pQueryInfo);
void tscResetForNextRetrieve(SSqlRes* pRes);
void executeQuery(SSqlObj* pSql, SQueryInfo* pQueryInfo);
@ -320,7 +322,7 @@ void tscTryQueryNextVnode(SSqlObj *pSql, __async_cb_func_t fp);
void tscAsyncQuerySingleRowForNextVnode(void *param, TAOS_RES *tres, int numOfRows);
void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp);
int tscSetMgmtEpSetFromCfg(const char *first, const char *second, SRpcCorEpSet *corEpSet);
int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVgroupNameList, __async_cb_func_t fp);
int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVgroupNameList, SArray* pUdfList, __async_cb_func_t fp);
int tscTransferTableNameList(SSqlObj *pSql, const char *pNameList, int32_t length, SArray* pNameArray);

1
src/client/inc/tsclient.h
View File

@ -177,6 +177,7 @@ typedef struct {
SInsertStatementParam insertParam;
char reserve1[3]; // fix bus error on arm32
int32_t count; // todo remove it
bool subCmd;
char reserve2[3]; // fix bus error on arm32
int16_t numOfCols;

59
src/client/src/tscGlobalmerge.c
View File

@ -602,6 +602,15 @@ static void doExecuteFinalMerge(SOperatorInfo* pOperator, int32_t numOfExpr, SSD
if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) {
continue;
}
if (functionId < 0) {
SUdfInfo* pUdfInfo = taosArrayGet(pInfo->udfInfo, -1 * functionId - 1);
doInvokeUdf(pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_MERGE);
continue;
}
aAggs[functionId].mergeFunc(&pCtx[j]);
}
} else {
@ -610,6 +619,15 @@ static void doExecuteFinalMerge(SOperatorInfo* pOperator, int32_t numOfExpr, SSD
if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) {
continue;
}
if (functionId < 0) {
SUdfInfo* pUdfInfo = taosArrayGet(pInfo->udfInfo, -1 * functionId - 1);
doInvokeUdf(pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE);
continue;
}
aAggs[functionId].xFinalize(&pCtx[j]);
}
@ -626,7 +644,11 @@ static void doExecuteFinalMerge(SOperatorInfo* pOperator, int32_t numOfExpr, SSD
}
for(int32_t j = 0; j < numOfExpr; ++j) {
aAggs[pCtx[j].functionId].init(&pCtx[j]);
if (pCtx[j].functionId < 0) {
continue;
}
aAggs[pCtx[j].functionId].init(&pCtx[j], pCtx[j].resultInfo);
}
for (int32_t j = 0; j < numOfExpr; ++j) {
@ -638,6 +660,15 @@ static void doExecuteFinalMerge(SOperatorInfo* pOperator, int32_t numOfExpr, SSD
if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) {
continue;
}
if (functionId < 0) {
SUdfInfo* pUdfInfo = taosArrayGet(pInfo->udfInfo, -1 * functionId - 1);
doInvokeUdf(pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_MERGE);
continue;
}
aAggs[functionId].mergeFunc(&pCtx[j]);
}
}
@ -651,6 +682,15 @@ static void doExecuteFinalMerge(SOperatorInfo* pOperator, int32_t numOfExpr, SSD
if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) {
continue;
}
if (functionId < 0) {
SUdfInfo* pUdfInfo = taosArrayGet(pInfo->udfInfo, -1 * functionId - 1);
doInvokeUdf(pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_MERGE);
continue;
}
aAggs[functionId].mergeFunc(&pCtx[j]);
}
}
@ -871,7 +911,6 @@ SSDataBlock* doGlobalAggregate(void* param, bool* newgroup) {
{
if (pAggInfo->hasDataBlockForNewGroup) {
pAggInfo->binfo.pRes->info.rows = 0;
pAggInfo->hasPrev = false; // now we start from a new group data set.
// not belongs to the same group, return the result of current group;
@ -880,7 +919,13 @@ SSDataBlock* doGlobalAggregate(void* param, bool* newgroup) {
{ // reset output buffer
for(int32_t j = 0; j < pOperator->numOfOutput; ++j) {
aAggs[pAggInfo->binfo.pCtx[j].functionId].init(&pAggInfo->binfo.pCtx[j]);
SQLFunctionCtx* pCtx = &pAggInfo->binfo.pCtx[j];
if (pCtx->functionId < 0) {
clearOutputBuf(&pAggInfo->binfo, &pAggInfo->bufCapacity);
continue;
}
aAggs[pCtx->functionId].init(pCtx, pCtx->resultInfo);
}
}
@ -933,6 +978,14 @@ SSDataBlock* doGlobalAggregate(void* param, bool* newgroup) {
continue;
}
if (functionId < 0) {
SUdfInfo* pUdfInfo = taosArrayGet(pAggInfo->udfInfo, -1 * functionId - 1);
doInvokeUdf(pUdfInfo, &pAggInfo->binfo.pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE);
continue;
}
aAggs[functionId].xFinalize(&pAggInfo->binfo.pCtx[j]);
}

576
src/client/src/tscSQLParser.c
View File

@ -37,6 +37,7 @@
#include "tstrbuild.h"
#include "ttoken.h"
#include "ttokendef.h"
#include "qScript.h"
#include "ttype.h"
#define DEFAULT_PRIMARY_TIMESTAMP_COL_NAME "_c0"
@ -77,7 +78,8 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC
static int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStrToken* tableName, int32_t* len);
static void getColumnName(tSqlExprItem* pItem, char* resultFieldName, char* rawName, int32_t nameLength);
static int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSqlExprItem* pItem, bool finalResult);
static int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSqlExprItem* pItem,
bool finalResult, SUdfInfo* pUdfInfo);
static int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnList* pIdList, int16_t bytes,
int8_t type, char* fieldName, SExprInfo* pSqlExpr);
@ -177,15 +179,15 @@ bool serializeExprListToVariant(SArray* pList, tVariant **dst, int16_t colType,
// check all the exprToken type in expr list same or not
if (firstVarType != var->nType) {
break;
}
}
if ((colType == TSDB_DATA_TYPE_BOOL || IS_SIGNED_NUMERIC_TYPE(colType))) {
if (var->nType != TSDB_DATA_TYPE_BOOL && !IS_SIGNED_NUMERIC_TYPE(var->nType)) {
break;
}
tbufWriteInt64(&bw, var->i64);
}
tbufWriteInt64(&bw, var->i64);
} else if (IS_UNSIGNED_NUMERIC_TYPE(colType)) {
if (IS_SIGNED_NUMERIC_TYPE(var->nType) || IS_UNSIGNED_NUMERIC_TYPE(var->nType)) {
tbufWriteUint64(&bw, var->u64);
tbufWriteUint64(&bw, var->u64);
} else {
break;
}
@ -216,19 +218,19 @@ bool serializeExprListToVariant(SArray* pList, tVariant **dst, int16_t colType,
} else if (colType == TSDB_DATA_TYPE_TIMESTAMP) {
if (var->nType == TSDB_DATA_TYPE_BINARY) {
if (convertTimestampStrToInt64(var, precision) < 0) {
break;
}
tbufWriteInt64(&bw, var->i64);
break;
}
tbufWriteInt64(&bw, var->i64);
} else if (var->nType == TSDB_DATA_TYPE_BIGINT) {
tbufWriteInt64(&bw, var->i64);
tbufWriteInt64(&bw, var->i64);
} else {
break;
}
}
} else {
break;
break;
}
if (i == (int32_t)(pList->size - 1)) { ret = true;}
}
}
if (ret == true) {
if ((*dst = calloc(1, sizeof(tVariant))) != NULL) {
tVariantCreateFromBinary(*dst, tbufGetData(&bw, false), tbufTell(&bw), TSDB_DATA_TYPE_BINARY);
@ -327,7 +329,7 @@ static int setColumnFilterInfoForTimestamp(SSqlCmd* pCmd, SQueryInfo* pQueryInfo
STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
if (convertTimestampStrToInt64(pVar, tinfo.precision) < -1) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg);
}
}
return TSDB_CODE_SUCCESS;
}
@ -386,6 +388,137 @@ static int32_t normalizeVarDataTypeLength(SSqlCmd* pCmd) {
return TSDB_CODE_SUCCESS;
}
int32_t readFromFile(char *name, uint32_t *len, void **buf) {
struct stat fileStat;
if (stat(name, &fileStat) < 0) {
tscError("stat file %s failed, error:%s", name, strerror(errno));
return TAOS_SYSTEM_ERROR(errno);
}
*len = fileStat.st_size;
if (*len <= 0) {
tscError("file %s is empty", name);
return TSDB_CODE_TSC_FILE_EMPTY;
}
*buf = calloc(1, *len);
if (*buf == NULL) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
int fd = open(name, O_RDONLY);
if (fd < 0) {
tscError("open file %s failed, error:%s", name, strerror(errno));
tfree(*buf);
return TAOS_SYSTEM_ERROR(errno);
}
int64_t s = taosRead(fd, *buf, *len);
if (s != *len) {
tscError("read file %s failed, error:%s", name, strerror(errno));
close(fd);
tfree(*buf);
return TSDB_CODE_TSC_APP_ERROR;
}
return TSDB_CODE_SUCCESS;
}
int32_t handleUserDefinedFunc(SSqlObj* pSql, struct SSqlInfo* pInfo) {
const char *msg1 = "function name is too long";
const char *msg2 = "path is too long";
const char *msg3 = "invalid outputtype";
const char *msg4 = "invalid script";
SSqlCmd *pCmd = &pSql->cmd;
switch (pInfo->type) {
case TSDB_SQL_CREATE_FUNCTION: {
SCreateFuncInfo *createInfo = &pInfo->pMiscInfo->funcOpt;
uint32_t len = 0;
void *buf = NULL;
if (createInfo->output.type == (uint8_t)-1 || createInfo->output.bytes < 0) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3);
}
createInfo->name.z[createInfo->name.n] = 0;
strdequote(createInfo->name.z);
if (strlen(createInfo->name.z) >= TSDB_FUNC_NAME_LEN) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
}
createInfo->path.z[createInfo->path.n] = 0;
strdequote(createInfo->path.z);
if (strlen(createInfo->path.z) >= PATH_MAX) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
int32_t ret = readFromFile(createInfo->path.z, &len, &buf);
if (ret) {
return ret;
}
//distinguish *.lua and *.so
int32_t pathLen = (int32_t)strlen(createInfo->path.z);
if ((pathLen > 3) && (0 == strncmp(createInfo->path.z + pathLen - 3, "lua", 3)) && !isValidScript(buf, len)) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4);
}
//TODO CHECK CODE
if (len + sizeof(SCreateFuncMsg) > pSql->cmd.allocSize) {
ret = tscAllocPayload(&pSql->cmd, len + sizeof(SCreateFuncMsg));
if (ret) {
tfree(buf);
return ret;
}
}
SCreateFuncMsg *pMsg = (SCreateFuncMsg *)pSql->cmd.payload;
strcpy(pMsg->name, createInfo->name.z);
strcpy(pMsg->path, createInfo->path.z);
pMsg->funcType = htonl(createInfo->type);
pMsg->bufSize = htonl(createInfo->bufSize);
pMsg->outputType = createInfo->output.type;
pMsg->outputLen = htons(createInfo->output.bytes);
pMsg->codeLen = htonl(len);
memcpy(pMsg->code, buf, len);
tfree(buf);
break;
}
case TSDB_SQL_DROP_FUNCTION: {
SStrToken* t0 = taosArrayGet(pInfo->pMiscInfo->a, 0);
SDropFuncMsg *pMsg = (SDropFuncMsg *)pSql->cmd.payload;
t0->z[t0->n] = 0;
strdequote(t0->z);
if (strlen(t0->z) >= TSDB_FUNC_NAME_LEN) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
}
strcpy(pMsg->name, t0->z);
break;
}
default:
return TSDB_CODE_TSC_APP_ERROR;
}
return TSDB_CODE_SUCCESS;
}
int32_t tscValidateSqlInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
if (pInfo == NULL || pSql == NULL) {
return TSDB_CODE_TSC_APP_ERROR;
@ -486,6 +619,16 @@ int32_t tscValidateSqlInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
break;
}
case TSDB_SQL_CREATE_FUNCTION:
case TSDB_SQL_DROP_FUNCTION: {
code = handleUserDefinedFunc(pSql, pInfo);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
break;
}
case TSDB_SQL_ALTER_DB:
case TSDB_SQL_CREATE_DB: {
const char* msg1 = "invalid db name";
@ -769,7 +912,15 @@ int32_t tscValidateSqlInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
return code;
}
SArray *pUdfInfo = NULL;
if (pQueryInfo->pUdfInfo) {
pUdfInfo = taosArrayDup(pQueryInfo->pUdfInfo);
}
pQueryInfo = pCmd->active;
pQueryInfo->pUdfInfo = pUdfInfo;
pQueryInfo->udfCopy = true;
}
}
@ -1018,7 +1169,7 @@ static int32_t validateStateWindowNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS
}
SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, index.columnIndex);
if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP || pSchema->type == TSDB_DATA_TYPE_FLOAT
if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP || pSchema->type == TSDB_DATA_TYPE_FLOAT
|| pSchema->type == TSDB_DATA_TYPE_DOUBLE || pSchema->type == TSDB_DATA_TYPE_NCHAR
|| pSchema->type == TSDB_DATA_TYPE_BINARY) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
@ -1813,6 +1964,69 @@ static bool hasNoneUserDefineExpr(SQueryInfo* pQueryInfo) {
return false;
}
void genUdfList(SArray* pUdfInfo, tSqlExpr *pNode) {
if (pNode == NULL) {
return;
}
if (pNode->type == SQL_NODE_EXPR) {
genUdfList(pUdfInfo, pNode->pLeft);
genUdfList(pUdfInfo, pNode->pRight);
return;
}
if (pNode->type == SQL_NODE_SQLFUNCTION) {
pNode->functionId = isValidFunction(pNode->Expr.operand.z, pNode->Expr.operand.n);
if (pNode->functionId < 0) { // extract all possible user defined function
struct SUdfInfo info = {0};
info.name = strndup(pNode->Expr.operand.z, pNode->Expr.operand.n);
int32_t functionId = (int32_t)taosArrayGetSize(pUdfInfo) * (-1) - 1;
info.functionId = functionId;
taosArrayPush(pUdfInfo, &info);
}
}
}
/*
static int32_t checkForUdf(SSqlObj* pSql, SQueryInfo* pQueryInfo, SArray* pSelection) {
if (pQueryInfo->pUdfInfo != NULL) {
return TSDB_CODE_SUCCESS;
}
pQueryInfo->pUdfInfo = taosArrayInit(4, sizeof(struct SUdfInfo));
size_t nExpr = taosArrayGetSize(pSelection);
for (int32_t i = 0; i < nExpr; ++i) {
tSqlExprItem* pItem = taosArrayGet(pSelection, i);
int32_t type = pItem->pNode->type;
if (type == SQL_NODE_EXPR || type == SQL_NODE_SQLFUNCTION) {
genUdfList(pQueryInfo->pUdfInfo, pItem->pNode);
}
}
if (taosArrayGetSize(pQueryInfo->pUdfInfo) > 0) {
return tscGetUdfFromNode(pSql, pQueryInfo);
} else {
return TSDB_CODE_SUCCESS;
}
}
*/
static SUdfInfo* isValidUdf(SArray* pUdfInfo, const char* name, int32_t len) {
size_t t = taosArrayGetSize(pUdfInfo);
for(int32_t i = 0; i < t; ++i) {
SUdfInfo* pUdf = taosArrayGet(pUdfInfo, i);
if (strlen(pUdf->name) == len && strncasecmp(pUdf->name, name, len) == 0) {
return pUdf;
}
}
return NULL;
}
int32_t validateSelectNodeList(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pSelNodeList, bool joinQuery,
bool timeWindowQuery, bool outerQuery) {
assert(pSelNodeList != NULL && pCmd != NULL);
@ -1845,12 +2059,18 @@ int32_t validateSelectNodeList(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pS
int32_t type = pItem->pNode->type;
if (type == SQL_NODE_SQLFUNCTION) {
pItem->pNode->functionId = isValidFunction(pItem->pNode->Expr.operand.z, pItem->pNode->Expr.operand.n);
SUdfInfo* pUdfInfo = NULL;
if (pItem->pNode->functionId < 0) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5);
pUdfInfo = isValidUdf(pQueryInfo->pUdfInfo, pItem->pNode->Expr.operand.z, pItem->pNode->Expr.operand.n);
if (pUdfInfo == NULL) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5);
}
pItem->pNode->functionId = pUdfInfo->functionId;
}
// sql function in selection clause, append sql function info in pSqlCmd structure sequentially
if (addExprAndResultField(pCmd, pQueryInfo, outputIndex, pItem, true) != TSDB_CODE_SUCCESS) {
if (addExprAndResultField(pCmd, pQueryInfo, outputIndex, pItem, true, pUdfInfo) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_OPERATION;
}
} else if (type == SQL_NODE_TABLE_COLUMN || type == SQL_NODE_VALUE) {
@ -2105,7 +2325,8 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t
}
static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSchema* pSchema, SConvertFunc cvtFunc,
const char* name, int32_t resColIdx, SColumnIndex* pColIndex, bool finalResult) {
const char* name, int32_t resColIdx, SColumnIndex* pColIndex, bool finalResult,
SUdfInfo* pUdfInfo) {
const char* msg1 = "not support column types";
int32_t f = cvtFunc.execFuncId;
@ -2121,7 +2342,7 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS
int16_t resBytes = 0;
int32_t interBufSize = 0;
getResultDataInfo(pSchema->type, pSchema->bytes, f, 0, &resType, &resBytes, &interBufSize, 0, false);
getResultDataInfo(pSchema->type, pSchema->bytes, f, 0, &resType, &resBytes, &interBufSize, 0, false, pUdfInfo);
SExprInfo* pExpr = tscExprAppend(pQueryInfo, f, pColIndex, resType, resBytes, getNewResColId(pCmd), interBufSize, false);
tstrncpy(pExpr->base.aliasName, name, tListLen(pExpr->base.aliasName));
@ -2193,18 +2414,19 @@ static void updateLastScanOrderIfNeeded(SQueryInfo* pQueryInfo) {
}
}
static UNUSED_FUNC void updateFunctionInterBuf(SQueryInfo* pQueryInfo, bool superTable) {
static UNUSED_FUNC void updateFunctionInterBuf(SQueryInfo* pQueryInfo, bool superTable, SUdfInfo* pUdfInfo) {
size_t numOfExpr = tscNumOfExprs(pQueryInfo);
for (int32_t i = 0; i < numOfExpr; ++i) {
SExprInfo* pExpr = tscExprGet(pQueryInfo, i);
int32_t param = (int32_t)pExpr->base.param[0].i64;
getResultDataInfo(pExpr->base.colType, pExpr->base.colBytes, pExpr->base.functionId, param, &pExpr->base.resType, &pExpr->base.resBytes,
&pExpr->base.interBytes, 0, superTable);
&pExpr->base.interBytes, 0, superTable, pUdfInfo);
}
}
int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSqlExprItem* pItem, bool finalResult) {
int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSqlExprItem* pItem, bool finalResult,
SUdfInfo* pUdfInfo) {
STableMetaInfo* pTableMetaInfo = NULL;
int32_t functionId = pItem->pNode->functionId;
@ -2353,7 +2575,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
int32_t intermediateResSize = 0;
if (getResultDataInfo(pSchema->type, pSchema->bytes, functionId, 0, &resultType, &resultSize,
&intermediateResSize, 0, false) != TSDB_CODE_SUCCESS) {
&intermediateResSize, 0, false, NULL) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_OPERATION;
}
@ -2487,7 +2709,8 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
SStrToken t = {.z = pSchema[j].name, .n = (uint32_t)strnlen(pSchema[j].name, TSDB_COL_NAME_LEN)};
setResultColName(name, pItem, cvtFunc.originFuncId, &t, true);
if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[j], cvtFunc, name, colIndex++, &index, finalResult) != 0) {
if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[j], cvtFunc, name, colIndex++, &index,
finalResult, pUdfInfo) != 0) {
return TSDB_CODE_TSC_INVALID_OPERATION;
}
}
@ -2510,7 +2733,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
bool multiColOutput = taosArrayGetSize(pItem->pNode->Expr.paramList) > 1;
setResultColName(name, pItem, cvtFunc.originFuncId, &pParamElem->pNode->columnName, multiColOutput);
if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, name, colIndex++, &index, finalResult) != 0) {
if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, name, colIndex++, &index, finalResult, pUdfInfo) != 0) {
return TSDB_CODE_TSC_INVALID_OPERATION;
}
}
@ -2535,7 +2758,8 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
SStrToken t = {.z = pSchema[i].name, .n = (uint32_t)strnlen(pSchema[i].name, TSDB_COL_NAME_LEN)};
setResultColName(name, pItem, cvtFunc.originFuncId, &t, true);
if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[index.columnIndex], cvtFunc, name, colIndex, &index, finalResult) != 0) {
if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[index.columnIndex], cvtFunc, name, colIndex, &index,
finalResult, pUdfInfo) != 0) {
return TSDB_CODE_TSC_INVALID_OPERATION;
}
colIndex++;
@ -2606,7 +2830,8 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5);
}
getResultDataInfo(pSchema->type, pSchema->bytes, functionId, 0, &resultType, &resultSize, &interResult, 0, false);
getResultDataInfo(pSchema->type, pSchema->bytes, functionId, 0, &resultType, &resultSize, &interResult, 0, false,
pUdfInfo);
/*
* sql function transformation
@ -2719,7 +2944,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
int16_t type = 0;
int32_t inter = 0;
int32_t ret = getResultDataInfo(s.type, s.bytes, TSDB_FUNC_TID_TAG, 0, &type, &bytes, &inter, 0, 0);
int32_t ret = getResultDataInfo(s.type, s.bytes, TSDB_FUNC_TID_TAG, 0, &type, &bytes, &inter, 0, 0, NULL);
assert(ret == TSDB_CODE_SUCCESS);
s.type = (uint8_t)type;
@ -2743,7 +2968,8 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
int32_t inter = 0;
int16_t resType = 0;
int16_t bytes = 0;
getResultDataInfo(TSDB_DATA_TYPE_INT, 4, TSDB_FUNC_BLKINFO, 0, &resType, &bytes, &inter, 0, 0);
getResultDataInfo(TSDB_DATA_TYPE_INT, 4, TSDB_FUNC_BLKINFO, 0, &resType, &bytes, &inter, 0, 0, NULL);
SSchema s = {.name = "block_dist", .type = TSDB_DATA_TYPE_BINARY, .bytes = bytes};
@ -2761,10 +2987,64 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
return TSDB_CODE_SUCCESS;
}
default:
return TSDB_CODE_TSC_INVALID_OPERATION;
default: {
pUdfInfo = isValidUdf(pQueryInfo->pUdfInfo, pItem->pNode->Expr.operand.z, pItem->pNode->Expr.operand.n);
if (pUdfInfo == NULL) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg9);
}
tSqlExprItem* pParamElem = taosArrayGet(pItem->pNode->Expr.paramList, 0);;
if (pParamElem->pNode->tokenId != TK_ID) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
SColumnIndex index = COLUMN_INDEX_INITIALIZER;
if (getColumnIndexByName(&pParamElem->pNode->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3);
}
if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6);
}
pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
// functions can not be applied to tags
if (index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6);
}
int32_t inter = 0;
int16_t resType = 0;
int16_t bytes = 0;
getResultDataInfo(TSDB_DATA_TYPE_INT, 4, functionId, 0, &resType, &bytes, &inter, 0, false, pUdfInfo);
SExprInfo* pExpr = tscExprAppend(pQueryInfo, functionId, &index, resType, bytes, getNewResColId(pCmd), inter, false);
memset(pExpr->base.aliasName, 0, tListLen(pExpr->base.aliasName));
getColumnName(pItem, pExpr->base.aliasName, pExpr->base.token, sizeof(pExpr->base.aliasName) - 1);
SSchema s = {0};
s.type = (uint8_t)resType;
s.bytes = bytes;
s.colId = pExpr->base.colInfo.colId;
uint64_t uid = pTableMetaInfo->pTableMeta->id.uid;
SColumnList ids = createColumnList(1, index.tableIndex, index.columnIndex);
if (finalResult) {
insertResultField(pQueryInfo, colIndex, &ids, pUdfInfo->resBytes, pUdfInfo->resType, pExpr->base.aliasName, pExpr);
} else {
for (int32_t i = 0; i < ids.num; ++i) {
tscColumnListInsert(pQueryInfo->colList, index.columnIndex, uid, &s);
}
}
tscInsertPrimaryTsSourceColumn(pQueryInfo, pTableMetaInfo->pTableMeta->id.uid);
return TSDB_CODE_SUCCESS;
}
}
return TSDB_CODE_TSC_INVALID_OPERATION;
}
// todo refactor
@ -3078,7 +3358,7 @@ int32_t tscTansformFuncForSTableQuery(SQueryInfo* pQueryInfo) {
(functionId >= TSDB_FUNC_FIRST_DST && functionId <= TSDB_FUNC_STDDEV_DST) ||
(functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_IRATE)) {
if (getResultDataInfo(pSrcSchema->type, pSrcSchema->bytes, functionId, (int32_t)pExpr->base.param[0].i64, &type, &bytes,
&interBytes, 0, true) != TSDB_CODE_SUCCESS) {
&interBytes, 0, true, NULL) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_OPERATION;
}
@ -3109,6 +3389,10 @@ void tscRestoreFuncForSTableQuery(SQueryInfo* pQueryInfo) {
int32_t inter = 0;
int32_t functionId = pExpr->base.functionId;
if (functionId < 0) {
continue;
}
if (functionId >= TSDB_FUNC_TS && functionId <= TSDB_FUNC_DIFF) {
continue;
}
@ -3120,9 +3404,9 @@ void tscRestoreFuncForSTableQuery(SQueryInfo* pQueryInfo) {
} else if (functionId == TSDB_FUNC_STDDEV_DST) {
functionId = TSDB_FUNC_STDDEV;
}
getResultDataInfo(pSchema->type, pSchema->bytes, functionId, 0, &pExpr->base.resType, &pExpr->base.resBytes,
&inter, 0, false);
getResultDataInfo(pSchema->type, pSchema->bytes, functionId, 0, &pExpr->base.resType, &pExpr->base.resBytes, &inter,
0, false, NULL);
}
}
@ -3135,6 +3419,10 @@ bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo)
size_t size = tscNumOfExprs(pQueryInfo);
for (int32_t i = 0; i < size; ++i) {
int32_t functionId = tscExprGet(pQueryInfo, i)->base.functionId;
if (functionId < 0) {
continue;
}
if ((aAggs[functionId].status & TSDB_FUNCSTATE_STABLE) == 0) {
invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3);
return true;
@ -3179,45 +3467,54 @@ static bool groupbyTagsOrNull(SQueryInfo* pQueryInfo) {
static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery, bool twQuery) {
int32_t startIdx = 0;
int32_t aggUdf = 0;
int32_t scalarUdf = 0;
int32_t prjNum = 0;
int32_t aggNum = 0;
size_t numOfExpr = tscNumOfExprs(pQueryInfo);
assert(numOfExpr > 0);
SExprInfo* pExpr = tscExprGet(pQueryInfo, startIdx);
// ts function can be simultaneously used with any other functions.
int32_t functionID = pExpr->base.functionId;
if (functionID == TSDB_FUNC_TS || functionID == TSDB_FUNC_TS_DUMMY) {
startIdx++;
}
int32_t factor = functionCompatList[tscExprGet(pQueryInfo, startIdx)->base.functionId];
if (tscExprGet(pQueryInfo, 0)->base.functionId == TSDB_FUNC_LAST_ROW && (joinQuery || twQuery || !groupbyTagsOrNull(pQueryInfo))) {
return false;
}
int32_t factor = INT32_MAX;
// diff function cannot be executed with other function
// arithmetic function can be executed with other arithmetic functions
size_t size = tscNumOfExprs(pQueryInfo);
for (int32_t i = startIdx + 1; i < size; ++i) {
for (int32_t i = startIdx; i < size; ++i) {
SExprInfo* pExpr1 = tscExprGet(pQueryInfo, i);
int16_t functionId = pExpr1->base.functionId;
if (functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TS) {
if (functionId < 0) {
SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1);
pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE ? ++aggUdf : ++scalarUdf;
continue;
}
if (functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY) {
++prjNum;
continue;
}
if (functionId == TSDB_FUNC_PRJ) {
++prjNum;
}
if (functionId == TSDB_FUNC_PRJ && (pExpr1->base.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX || TSDB_COL_IS_UD_COL(pExpr1->base.colInfo.flag))) {
continue;
}
if (functionCompatList[functionId] != factor) {
return false;
if (factor == INT32_MAX) {
factor = functionCompatList[functionId];
} else {
if (factor == -1) { // two functions with the same -1 flag
if (functionCompatList[functionId] != factor) {
return false;
} else {
if (factor == -1) { // two functions with the same -1 flag
return false;
}
}
}
@ -3226,6 +3523,18 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery, bool
}
}
aggNum = (int32_t)size - prjNum - aggUdf - scalarUdf;
assert(aggNum >= 0);
if (aggUdf > 0 && (prjNum > 0 || aggNum > 0 || scalarUdf > 0)) {
return false;
}
if (scalarUdf > 0 && aggNum > 0) {
return false;
}
return true;
}
@ -3291,7 +3600,7 @@ int32_t validateGroupbyNode(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd
pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
STableMeta* pTableMeta = pTableMetaInfo->pTableMeta;
if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) {
pSchema = tGetTbnameColumnSchema();
} else {
@ -3300,7 +3609,7 @@ int32_t validateGroupbyNode(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd
int32_t numOfCols = tscGetNumOfColumns(pTableMeta);
bool groupTag = (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX || index.columnIndex >= numOfCols);
if (groupTag) {
if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6);
@ -3421,7 +3730,7 @@ static int32_t doExtractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo,
pColumnFilter->pz = (int64_t)calloc(1, pVal->nLen + 1);
pColumnFilter->len = pVal->nLen;
pColumnFilter->filterstr = 1;
memcpy((char *)(pColumnFilter->pz), (char *)(pVal->pz), pVal->nLen);
memcpy((char *)(pColumnFilter->pz), (char *)(pVal->pz), pVal->nLen);
tVariantDestroy(pVal);
free(pVal);
@ -3834,7 +4143,7 @@ static int32_t validateSQLExpr(SSqlCmd* pCmd, tSqlExpr* pExpr, SQueryInfo* pQuer
return TSDB_CODE_TSC_INVALID_OPERATION;
}
if (addExprAndResultField(pCmd, pQueryInfo, outputIndex, &item, false) != TSDB_CODE_SUCCESS) {
if (addExprAndResultField(pCmd, pQueryInfo, outputIndex, &item, false, NULL) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_OPERATION;
}
@ -4877,7 +5186,7 @@ int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSq
if ((ret = getColumnQueryCondInfo(&pSql->cmd, pQueryInfo, condExpr.pColumnCond, TK_AND)) != TSDB_CODE_SUCCESS) {
goto PARSE_WHERE_EXIT;
}
if (taosArrayGetSize(pQueryInfo->pUpstream) > 0 ) {
if ((ret = getColumnQueryCondInfo(&pSql->cmd, pQueryInfo, condExpr.pTimewindow, TK_AND)) != TSDB_CODE_SUCCESS) {
goto PARSE_WHERE_EXIT;
@ -5237,14 +5546,14 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq
if (PRIMARYKEY_TIMESTAMP_COL_INDEX == index.columnIndex) {
orderByTS = true;
}
SArray *columnInfo = pQueryInfo->groupbyExpr.columnInfo;
if (columnInfo != NULL && taosArrayGetSize(columnInfo) > 0) {
SColIndex* pColIndex = taosArrayGet(columnInfo, 0);
if (PRIMARYKEY_TIMESTAMP_COL_INDEX != index.columnIndex && pColIndex->colIndex == index.columnIndex) {
orderByGroupbyCol = true;
orderByGroupbyCol = true;
}
}
}
if (!(orderByTags || orderByTS || orderByGroupbyCol) && !isTopBottomQuery(pQueryInfo)) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3);
} else { // order by top/bottom result value column is not supported in case of interval query.
@ -5328,15 +5637,15 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq
SArray *columnInfo = pQueryInfo->groupbyExpr.columnInfo;
if (columnInfo != NULL && taosArrayGetSize(columnInfo) > 0) {
SColIndex* pColIndex = taosArrayGet(columnInfo, 0);
validOrder = (pColIndex->colIndex == index.columnIndex);
}
validOrder = (pColIndex->colIndex == index.columnIndex);
}
if (!validOrder) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
}
tVariantListItem* p1 = taosArrayGet(pSqlNode->pSortOrder, 0);
pQueryInfo->groupbyExpr.orderIndex = pSchema[index.columnIndex].colId;
pQueryInfo->groupbyExpr.orderType = p1->sortOrder;
pQueryInfo->groupbyExpr.orderIndex = pSchema[index.columnIndex].colId;
pQueryInfo->groupbyExpr.orderType = p1->sortOrder;
}
if (isTopBottomQuery(pQueryInfo)) {
@ -5344,7 +5653,7 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq
SArray *columnInfo = pQueryInfo->groupbyExpr.columnInfo;
if (columnInfo != NULL && taosArrayGetSize(columnInfo) > 0) {
SColIndex* pColIndex = taosArrayGet(columnInfo, 0);
validOrder = (pColIndex->colIndex == index.columnIndex);
validOrder = (pColIndex->colIndex == index.columnIndex);
} else {
/* order of top/bottom query in interval is not valid */
SExprInfo* pExpr = tscExprGet(pQueryInfo, 0);
@ -5354,16 +5663,16 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq
if (pExpr->base.colInfo.colIndex != index.columnIndex && index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
validOrder = true;
}
validOrder = true;
}
if (!validOrder) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
}
tVariantListItem* pItem = taosArrayGet(pSqlNode->pSortOrder, 0);
pQueryInfo->order.order = pItem->sortOrder;
pQueryInfo->order.orderColId = pSchema[index.columnIndex].colId;
return TSDB_CODE_SUCCESS;
}
@ -5775,6 +6084,16 @@ int32_t validateFunctionsInIntervalOrGroupbyQuery(SSqlCmd* pCmd, SQueryInfo* pQu
for (int32_t k = 0; k < size; ++k) {
SExprInfo* pExpr = tscExprGet(pQueryInfo, k);
if (pExpr->base.functionId < 0) {
SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * pExpr->base.functionId - 1);
if (pUdfInfo->funcType == TSDB_UDF_TYPE_SCALAR) {
isProjectionFunction = true;
break;
} else {
continue;
}
}
// projection query on primary timestamp, the selectivity function needs to be present.
if (pExpr->base.functionId == TSDB_FUNC_PRJ && pExpr->base.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) {
bool hasSelectivity = false;
@ -5794,6 +6113,7 @@ int32_t validateFunctionsInIntervalOrGroupbyQuery(SSqlCmd* pCmd, SQueryInfo* pQu
int32_t f = pExpr->base.functionId;
if ((f == TSDB_FUNC_PRJ && pExpr->base.numOfParams == 0) || f == TSDB_FUNC_DIFF || f == TSDB_FUNC_ARITHM || f == TSDB_FUNC_DERIVATIVE) {
isProjectionFunction = true;
break;
}
}
@ -6279,11 +6599,15 @@ static void doUpdateSqlFunctionForTagPrj(SQueryInfo* pQueryInfo) {
for (int32_t i = 0; i < size; ++i) {
SExprInfo* pExpr = tscExprGet(pQueryInfo, i);
if (pExpr->base.functionId < 0) {
continue;
}
if ((pExpr->base.functionId != TSDB_FUNC_TAG_DUMMY && pExpr->base.functionId != TSDB_FUNC_TS_DUMMY) &&
!(pExpr->base.functionId == TSDB_FUNC_PRJ && TSDB_COL_IS_UD_COL(pExpr->base.colInfo.flag))) {
SSchema* pColSchema = &pSchema[pExpr->base.colInfo.colIndex];
getResultDataInfo(pColSchema->type, pColSchema->bytes, pExpr->base.functionId, (int32_t)pExpr->base.param[0].i64, &pExpr->base.resType,
&pExpr->base.resBytes, &pExpr->base.interBytes, tagLength, isSTable);
&pExpr->base.resBytes, &pExpr->base.interBytes, tagLength, isSTable, NULL);
}
}
}
@ -6411,6 +6735,14 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, char* msg) {
continue;
}
if (functionId < 0) {
SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1);
if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) {
++numOfAggregation;
}
continue;
}
if ((aAggs[functionId].status & TSDB_FUNCSTATE_SELECTIVITY) != 0) {
numOfSelectivity++;
@ -6638,6 +6970,10 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, char*
}
}
if (f < 0) {
continue;
}
if ((!pQueryInfo->stateWindow) && (f == TSDB_FUNC_DIFF || f == TSDB_FUNC_DERIVATIVE || f == TSDB_FUNC_TWA || f == TSDB_FUNC_IRATE)) {
for (int32_t j = 0; j < pQueryInfo->groupbyExpr.numOfGroupCols; ++j) {
SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, j);
@ -6646,13 +6982,13 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, char*
return invalidOperationMsg(msg, msg6);
}
} else if (!TSDB_COL_IS_TAG(pColIndex->flag)) {
return invalidOperationMsg(msg, msg6);
return invalidOperationMsg(msg, msg6);
}
}
}
if (IS_MULTIOUTPUT(aAggs[f].status) && f != TSDB_FUNC_TOP && f != TSDB_FUNC_BOTTOM &&
f != TSDB_FUNC_DIFF && f != TSDB_FUNC_DERIVATIVE && f != TSDB_FUNC_TAGPRJ && f != TSDB_FUNC_PRJ) {
if (IS_MULTIOUTPUT(aAggs[f].status) && f != TSDB_FUNC_TOP && f != TSDB_FUNC_BOTTOM && f != TSDB_FUNC_DIFF &&
f != TSDB_FUNC_DERIVATIVE && f != TSDB_FUNC_TAGPRJ && f != TSDB_FUNC_PRJ) {
return invalidOperationMsg(msg, msg1);
}
@ -6679,6 +7015,7 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, char*
return checkUpdateTagPrjFunctions(pQueryInfo, msg);
}
}
int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode) {
const char* msg1 = "only one expression allowed";
const char* msg2 = "invalid expression in select clause";
@ -6857,9 +7194,17 @@ void tscPrintSelNodeList(SSqlObj* pSql, int32_t subClauseIndex) {
char tmpBuf[1024] = {0};
int32_t tmpLen = 0;
char *name = NULL;
if (pExpr->base.functionId < 0) {
SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * pExpr->base.functionId - 1);
name = pUdfInfo->name;
} else {
name = aAggs[pExpr->base.functionId].name;
}
tmpLen =
sprintf(tmpBuf, "%s(uid:%" PRIu64 ", %d)", aAggs[pExpr->base.functionId].name, pExpr->base.uid,
pExpr->base.colInfo.colId);
sprintf(tmpBuf, "%s(uid:%" PRId64 ", %d)", name, pExpr->base.uid, pExpr->base.colInfo.colId);
if (tmpLen + offset >= totalBufSize - 1) break;
@ -6951,7 +7296,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
const int32_t STABLE_INDEX = 1;
STableMetaInfo* pStableMetaInfo = tscGetMetaInfo(pQueryInfo, STABLE_INDEX);
// super table name, create table by using dst
int32_t numOfTables = (int32_t) taosArrayGetSize(pCreateTable->childTableInfo);
for(int32_t j = 0; j < numOfTables; ++j) {
@ -7037,7 +7382,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4);
}
} else if (pItem->pVar.nType == TSDB_DATA_TYPE_TIMESTAMP) {
pItem->pVar.i64 = convertTimePrecision(pItem->pVar.i64, TSDB_TIME_PRECISION_NANO, tinfo.precision);
pItem->pVar.i64 = convertTimePrecision(pItem->pVar.i64, TSDB_TIME_PRECISION_NANO, tinfo.precision);
}
}
@ -7092,10 +7437,10 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4);
}
} else if (pItem->pVar.nType == TSDB_DATA_TYPE_TIMESTAMP) {
pItem->pVar.i64 = convertTimePrecision(pItem->pVar.i64, TSDB_TIME_PRECISION_NANO, tinfo.precision);
pItem->pVar.i64 = convertTimePrecision(pItem->pVar.i64, TSDB_TIME_PRECISION_NANO, tinfo.precision);
}
}
ret = tVariantDump(&(pItem->pVar), tagVal, pSchema->type, true);
@ -7388,7 +7733,7 @@ int32_t tscGetExprFilters(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pSelect
int32_t outputIndex = (int32_t)tscNumOfExprs(pQueryInfo);
// ADD TRUE FOR TEST
if (addExprAndResultField(pCmd, pQueryInfo, outputIndex, &item, true) != TSDB_CODE_SUCCESS) {
if (addExprAndResultField(pCmd, pQueryInfo, outputIndex, &item, true, NULL) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_OPERATION;
}
@ -7678,6 +8023,7 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) {
SArray* pVgroupList = NULL;
SArray* plist = NULL;
STableMeta* pTableMeta = NULL;
SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd);
pCmd->pTableMetaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
@ -7761,9 +8107,41 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) {
}
}
size_t funcSize = 0;
if (pInfo->funcs) {
funcSize = taosArrayGetSize(pInfo->funcs);
}
if (funcSize > 0) {
for (size_t i = 0; i < funcSize; ++i) {
SStrToken* t = taosArrayGet(pInfo->funcs, i);
if (NULL == t) {
continue;
}
if (t->n >= TSDB_FUNC_NAME_LEN) {
code = tscSQLSyntaxErrMsg(tscGetErrorMsgPayload(pCmd), "too long function name", t->z);
if (code != TSDB_CODE_SUCCESS) {
goto _end;
}
}
int32_t functionId = isValidFunction(t->z, t->n);
if (functionId < 0) {
struct SUdfInfo info = {0};
info.name = strndup(t->z, t->n);
if (pQueryInfo->pUdfInfo == NULL) {
pQueryInfo->pUdfInfo = taosArrayInit(4, sizeof(struct SUdfInfo));
}
info.functionId = (int32_t)taosArrayGetSize(pQueryInfo->pUdfInfo) * (-1) - 1;;
taosArrayPush(pQueryInfo->pUdfInfo, &info);
}
}
}
// load the table meta for a given table name list
if (taosArrayGetSize(plist) > 0 || taosArrayGetSize(pVgroupList) > 0) {
code = getMultiTableMetaFromMnode(pSql, plist, pVgroupList, tscTableMetaCallBack);
if (taosArrayGetSize(plist) > 0 || taosArrayGetSize(pVgroupList) > 0 || (pQueryInfo->pUdfInfo && taosArrayGetSize(pQueryInfo->pUdfInfo) > 0)) {
code = getMultiTableMetaFromMnode(pSql, plist, pVgroupList, pQueryInfo->pUdfInfo, tscTableMetaCallBack);
}
_end:
@ -7896,8 +8274,15 @@ static int32_t doValidateSubquery(SSqlNode* pSqlNode, int32_t index, SSqlObj* pS
SQueryInfo* pSub = calloc(1, sizeof(SQueryInfo));
tscInitQueryInfo(pSub);
SArray *pUdfInfo = NULL;
if (pQueryInfo->pUdfInfo) {
pUdfInfo = taosArrayDup(pQueryInfo->pUdfInfo);
}
pSub->pUdfInfo = pUdfInfo;
pSub->udfCopy = true;
int32_t code = validateSqlNode(pSql, p, pSub);
assert(code != TSDB_CODE_TSC_ACTION_IN_PROGRESS);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
@ -8234,11 +8619,12 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf
}
{ // set the query info
pQueryInfo->projectionQuery = tscIsProjectionQuery(pQueryInfo);
pQueryInfo->hasFilter = tscHasColumnFilter(pQueryInfo);
pQueryInfo->simpleAgg = isSimpleAggregateRv(pQueryInfo);
pQueryInfo->onlyTagQuery = onlyTagPrjFunction(pQueryInfo);
pQueryInfo->groupbyColumn = tscGroupbyColumn(pQueryInfo);
pQueryInfo->projectionQuery = tscIsProjectionQuery(pQueryInfo);
pQueryInfo->hasFilter = tscHasColumnFilter(pQueryInfo);
pQueryInfo->simpleAgg = isSimpleAggregateRv(pQueryInfo);
pQueryInfo->onlyTagQuery = onlyTagPrjFunction(pQueryInfo);
pQueryInfo->groupbyColumn = tscGroupbyColumn(pQueryInfo);
//pQueryInfo->globalMerge = tscIsTwoStageSTableQuery(pQueryInfo, 0);
pQueryInfo->arithmeticOnAgg = tsIsArithmeticQueryOnAggResult(pQueryInfo);
pQueryInfo->orderProjectQuery = tscOrderedProjectionQueryOnSTable(pQueryInfo, 0);
@ -8294,7 +8680,7 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlExpr* pS
assert(pSqlExpr->pRight == NULL);
if (pSqlExpr->type == SQL_NODE_VALUE) {
int32_t ret = TSDB_CODE_SUCCESS;
int32_t ret = TSDB_CODE_SUCCESS;
*pExpr = calloc(1, sizeof(tExprNode));
(*pExpr)->nodeType = TSQL_NODE_VALUE;
(*pExpr)->pVal = calloc(1, sizeof(tVariant));
@ -8304,12 +8690,12 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlExpr* pS
if (pCols != NULL && taosArrayGetSize(pCols) > 0) {
SColIndex* idx = taosArrayGet(pCols, 0);
SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, idx->colIndex);
// convert time by precision
// convert time by precision
if (pSchema != NULL && TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && TSDB_DATA_TYPE_BINARY == (*pExpr)->pVal->nType) {
ret = setColumnFilterInfoForTimestamp(pCmd, pQueryInfo, (*pExpr)->pVal);
}
}
return ret;
return ret;
} else if (pSqlExpr->type == SQL_NODE_SQLFUNCTION) {
// arithmetic expression on the results of aggregation functions
*pExpr = calloc(1, sizeof(tExprNode));
@ -8468,9 +8854,7 @@ void normalizeSqlNode(SSqlNode* pSqlNode, const char* dbName) {
// 2. pSqlNode->pWhere
// 3. pSqlNode->pHaving
// 4. pSqlNode->pSortOrder
// pSqlNode->from
}
#endif

257
src/client/src/tscServer.c
View File

@ -503,6 +503,7 @@ int doBuildAndSendMsg(SSqlObj *pSql) {
pCmd->command == TSDB_SQL_INSERT ||
pCmd->command == TSDB_SQL_CONNECT ||
pCmd->command == TSDB_SQL_HB ||
pCmd->command == TSDB_SQL_RETRIEVE_FUNC ||
pCmd->command == TSDB_SQL_STABLEVGROUP) {
pRes->code = tscBuildMsg[pCmd->command](pSql, NULL);
}
@ -545,7 +546,7 @@ int tscBuildAndSendRequest(SSqlObj *pSql, SQueryInfo* pQueryInfo) {
type = pQueryInfo->type;
// while numOfTables equals to 0, it must be Heartbeat
assert((pQueryInfo->numOfTables == 0 && pQueryInfo->command == TSDB_SQL_HB) || pQueryInfo->numOfTables > 0);
assert((pQueryInfo->numOfTables == 0 && (pQueryInfo->command == TSDB_SQL_HB || pSql->cmd.command == TSDB_SQL_RETRIEVE_FUNC)) || pQueryInfo->numOfTables > 0);
}
tscDebug("0x%"PRIx64" SQL cmd:%s will be processed, name:%s, type:%d", pSql->self, sqlCmd[pCmd->command], name, type);
@ -1032,6 +1033,34 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pMsg += sizeof(int32_t);
}
// support only one udf
if (pQueryInfo->pUdfInfo != NULL && taosArrayGetSize(pQueryInfo->pUdfInfo) > 0) {
pQueryMsg->udfContentOffset = htonl((int32_t) (pMsg - pCmd->payload));
for(int32_t i = 0; i < taosArrayGetSize(pQueryInfo->pUdfInfo); ++i) {
SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, i);
*(int8_t*) pMsg = pUdfInfo->resType;
pMsg += sizeof(pUdfInfo->resType);
*(int16_t*) pMsg = htons(pUdfInfo->resBytes);
pMsg += sizeof(pUdfInfo->resBytes);
STR_TO_VARSTR(pMsg, pUdfInfo->name);
pMsg += varDataTLen(pMsg);
*(int32_t*) pMsg = htonl(pUdfInfo->funcType);
pMsg += sizeof(pUdfInfo->funcType);
*(int32_t*) pMsg = htonl(pUdfInfo->bufSize);
pMsg += sizeof(pUdfInfo->bufSize);
pQueryMsg->udfContentLen = htonl(pUdfInfo->contLen);
memcpy(pMsg, pUdfInfo->content, pUdfInfo->contLen);
pMsg += pUdfInfo->contLen;
}
}
memcpy(pMsg, pSql->sqlstr, sqlLen);
pMsg += sqlLen;
@ -1067,6 +1096,18 @@ int32_t tscBuildCreateDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
return TSDB_CODE_SUCCESS;
}
int32_t tscBuildCreateFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
SCreateFuncMsg *pCreateFuncMsg = (SCreateFuncMsg *)pCmd->payload;
pCmd->msgType = TSDB_MSG_TYPE_CM_CREATE_FUNCTION;
pCmd->payloadLen = sizeof(SCreateFuncMsg) + htonl(pCreateFuncMsg->codeLen);
return TSDB_CODE_SUCCESS;
}
int32_t tscBuildCreateDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
pCmd->payloadLen = sizeof(SCreateDnodeMsg);
@ -1191,6 +1232,17 @@ int32_t tscBuildDropDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
return TSDB_CODE_SUCCESS;
}
int32_t tscBuildDropFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_FUNCTION;
pCmd->payloadLen = sizeof(SDropFuncMsg);
return TSDB_CODE_SUCCESS;
}
int32_t tscBuildDropTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
pCmd->payloadLen = sizeof(SCMDropTableMsg);
@ -1294,11 +1346,19 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
SShowMsg *pShowMsg = (SShowMsg *)pCmd->payload;
SShowInfo *pShowInfo = &pInfo->pMiscInfo->showOpt;
SShowMsg *pShowMsg = (SShowMsg *)pCmd->payload;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0);
if (pShowInfo->showType == TSDB_MGMT_TABLE_FUNCTION) {
pShowMsg->type = pShowInfo->showType;
pShowMsg->payloadLen = 0;
pCmd->payloadLen = sizeof(SShowMsg);
if (tNameIsEmpty(&pTableMetaInfo->name)) {
return TSDB_CODE_SUCCESS;
}
if (tNameIsEmpty(&pTableMetaInfo->name)) {
pthread_mutex_lock(&pObj->mutex);
tstrncpy(pShowMsg->db, pObj->db, sizeof(pShowMsg->db));
pthread_mutex_unlock(&pObj->mutex);
@ -1306,7 +1366,6 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
tNameGetFullDbName(&pTableMetaInfo->name, pShowMsg->db);
}
SShowInfo *pShowInfo = &pInfo->pMiscInfo->showOpt;
pShowMsg->type = pShowInfo->showType;
if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) {
@ -1819,6 +1878,29 @@ int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
return TSDB_CODE_SUCCESS;
}
int tscBuildRetrieveFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
char *pMsg = pCmd->payload;
SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd);
int32_t numOfFuncs = (int32_t)taosArrayGetSize(pQueryInfo->pUdfInfo);
SRetrieveFuncMsg *pRetrieveFuncMsg = (SRetrieveFuncMsg *)pMsg;
pRetrieveFuncMsg->num = htonl(numOfFuncs);
pMsg += sizeof(SRetrieveFuncMsg);
for(int32_t i = 0; i < numOfFuncs; ++i) {
SUdfInfo* pUdf = taosArrayGet(pQueryInfo->pUdfInfo, i);
STR_TO_NET_VARSTR(pMsg, pUdf->name);
pMsg += varDataNetTLen(pMsg);
}
pCmd->msgType = TSDB_MSG_TYPE_CM_RETRIEVE_FUNC;
pCmd->payloadLen = (int32_t)(pMsg - pCmd->payload);
return TSDB_CODE_SUCCESS;
}
int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
STscObj *pObj = pSql->pTscObj;
@ -2034,12 +2116,64 @@ static SVgroupsInfo* createVgroupInfoFromMsg(char* pMsg, int32_t* size, uint64_t
return pVgroupInfo;
}
int tscProcessRetrieveFuncRsp(SSqlObj* pSql) {
SSqlCmd* pCmd = &pSql->cmd;
SUdfFuncMsg* pFuncMsg = (SUdfFuncMsg *)pSql->res.pRsp;
SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd);
pFuncMsg->num = htonl(pFuncMsg->num);
assert(pFuncMsg->num == taosArrayGetSize(pQueryInfo->pUdfInfo));
char* pMsg = pFuncMsg->content;
for(int32_t i = 0; i < pFuncMsg->num; ++i) {
SFunctionInfoMsg* pFunc = (SFunctionInfoMsg*) pMsg;
for(int32_t j = 0; j < pFuncMsg->num; ++j) {
SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, j);
if (strcmp(pUdfInfo->name, pFunc->name) != 0) {
continue;
}
if (pUdfInfo->content) {
continue;
}
pUdfInfo->resBytes = htons(pFunc->resBytes);
pUdfInfo->resType = pFunc->resType;
pUdfInfo->funcType = htonl(pFunc->funcType);
pUdfInfo->contLen = htonl(pFunc->len);
pUdfInfo->bufSize = htonl(pFunc->bufSize);
pUdfInfo->content = malloc(pUdfInfo->contLen);
memcpy(pUdfInfo->content, pFunc->content, pUdfInfo->contLen);
pMsg += sizeof(SFunctionInfoMsg) + pUdfInfo->contLen;
}
}
// master sqlObj locates in param
SSqlObj* parent = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)pSql->param);
if(parent == NULL) {
return pSql->res.code;
}
SQueryInfo* parQueryInfo = tscGetQueryInfo(&parent->cmd);
assert(parent->signature == parent && (int64_t)pSql->param == parent->self);
taosArrayDestroy(parQueryInfo->pUdfInfo);
parQueryInfo->pUdfInfo = pQueryInfo->pUdfInfo; // assigned to parent sql obj.
pQueryInfo->pUdfInfo = NULL;
return TSDB_CODE_SUCCESS;
}
int tscProcessMultiTableMetaRsp(SSqlObj *pSql) {
char *rsp = pSql->res.pRsp;
SMultiTableMeta *pMultiMeta = (SMultiTableMeta *)rsp;
pMultiMeta->numOfTables = htonl(pMultiMeta->numOfTables);
pMultiMeta->numOfVgroup = htonl(pMultiMeta->numOfVgroup);
pMultiMeta->numOfUdf = htonl(pMultiMeta->numOfUdf);
rsp += sizeof(SMultiTableMeta);
@ -2128,6 +2262,37 @@ int tscProcessMultiTableMetaRsp(SSqlObj *pSql) {
pMsg += size;
}
SQueryInfo* pQueryInfo = tscGetQueryInfo(pParentCmd);
if (pMultiMeta->numOfUdf > 0) {
assert(pQueryInfo->pUdfInfo != NULL);
}
for(int32_t i = 0; i < pMultiMeta->numOfUdf; ++i) {
SFunctionInfoMsg* pFunc = (SFunctionInfoMsg*) pMsg;
for(int32_t j = 0; j < pMultiMeta->numOfUdf; ++j) {
SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, j);
if (strcmp(pUdfInfo->name, pFunc->name) != 0) {
continue;
}
if (pUdfInfo->content) {
continue;
}
pUdfInfo->resBytes = htons(pFunc->resBytes);
pUdfInfo->resType = pFunc->resType;
pUdfInfo->funcType = htonl(pFunc->funcType);
pUdfInfo->contLen = htonl(pFunc->len);
pUdfInfo->bufSize = htonl(pFunc->bufSize);
pUdfInfo->content = malloc(pUdfInfo->contLen);
memcpy(pUdfInfo->content, pFunc->content, pUdfInfo->contLen);
pMsg += sizeof(SFunctionInfoMsg) + pUdfInfo->contLen;
}
}
pSql->res.code = TSDB_CODE_SUCCESS;
pSql->res.numOfTotal = pMultiMeta->numOfTables;
tscDebug("0x%"PRIx64" load multi-tableMeta from mnode, numOfTables:%d", pSql->self, pMultiMeta->numOfTables);
@ -2518,7 +2683,7 @@ static int32_t getTableMetaFromMnode(SSqlObj *pSql, STableMetaInfo *pTableMetaIn
return code;
}
int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVgroupNameList, __async_cb_func_t fp) {
int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVgroupNameList, SArray* pUdfList, __async_cb_func_t fp) {
SSqlObj *pNew = calloc(1, sizeof(SSqlObj));
if (NULL == pNew) {
tscError("0x%"PRIx64" failed to allocate sqlobj to get multiple table meta", pSql->self);
@ -2531,8 +2696,9 @@ int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVg
int32_t numOfTable = (int32_t) taosArrayGetSize(pNameList);
int32_t numOfVgroupList = (int32_t) taosArrayGetSize(pVgroupNameList);
int32_t numOfUdf = pUdfList ? (int32_t)taosArrayGetSize(pUdfList) : 0;
int32_t size = (numOfTable + numOfVgroupList) * TSDB_TABLE_FNAME_LEN + sizeof(SMultiTableInfoMsg);
int32_t size = (numOfTable + numOfVgroupList) * TSDB_TABLE_FNAME_LEN + TSDB_FUNC_NAME_LEN * numOfUdf + sizeof(SMultiTableInfoMsg);
if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, size)) {
tscError("0x%"PRIx64" malloc failed for payload to get table meta", pSql->self);
tscFreeSqlObj(pNew);
@ -2542,12 +2708,13 @@ int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVg
SMultiTableInfoMsg* pInfo = (SMultiTableInfoMsg*) pNew->cmd.payload;
pInfo->numOfTables = htonl((uint32_t) taosArrayGetSize(pNameList));
pInfo->numOfVgroups = htonl((uint32_t) taosArrayGetSize(pVgroupNameList));
pInfo->numOfUdfs = htonl(numOfUdf);
char* start = pInfo->tableNames;
int32_t len = 0;
for(int32_t i = 0; i < numOfTable; ++i) {
char* name = taosArrayGetP(pNameList, i);
if (i < numOfTable - 1 || numOfVgroupList > 0) {
if (i < numOfTable - 1 || numOfVgroupList > 0 || numOfUdf > 0) {
len = sprintf(start, "%s,", name);
} else {
len = sprintf(start, "%s", name);
@ -2558,7 +2725,7 @@ int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVg
for(int32_t i = 0; i < numOfVgroupList; ++i) {
char* name = taosArrayGetP(pVgroupNameList, i);
if (i < numOfVgroupList - 1) {
if (i < numOfVgroupList - 1 || numOfUdf > 0) {
len = sprintf(start, "%s,", name);
} else {
len = sprintf(start, "%s", name);
@ -2567,12 +2734,23 @@ int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVg
start += len;
}
for(int32_t i = 0; i < numOfUdf; ++i) {
SUdfInfo * u = taosArrayGet(pUdfList, i);
if (i < numOfUdf - 1) {
len = sprintf(start, "%s,", u->name);
} else {
len = sprintf(start, "%s", u->name);
}
start += len;
}
pNew->cmd.payloadLen = (int32_t) ((start - pInfo->tableNames) + sizeof(SMultiTableInfoMsg));
pNew->cmd.msgType = TSDB_MSG_TYPE_CM_TABLES_META;
registerSqlObj(pNew);
tscDebug("0x%"PRIx64" new pSqlObj:0x%"PRIx64" to get %d tableMeta, vgroupInfo:%d, msg size:%d", pSql->self,
pNew->self, numOfTable, numOfVgroupList, pNew->cmd.payloadLen);
tscDebug("0x%"PRIx64" new pSqlObj:0x%"PRIx64" to get %d tableMeta, vgroupInfo:%d, udf:%d, msg size:%d", pSql->self,
pNew->self, numOfTable, numOfVgroupList, numOfUdf, pNew->cmd.payloadLen);
pNew->fp = fp;
pNew->param = (void *)pSql->self;
@ -2645,6 +2823,60 @@ int tscGetTableMetaEx(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool create
return tscGetTableMetaImpl(pSql, pTableMetaInfo, createIfNotExists);
}
int32_t tscGetUdfFromNode(SSqlObj *pSql, SQueryInfo* pQueryInfo) {
SSqlObj *pNew = calloc(1, sizeof(SSqlObj));
if (NULL == pNew) {
tscError("%p malloc failed for new sqlobj to get user-defined functions", pSql);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
pNew->pTscObj = pSql->pTscObj;
pNew->signature = pNew;
pNew->cmd.command = TSDB_SQL_RETRIEVE_FUNC;
if (tscAddQueryInfo(&pNew->cmd) != TSDB_CODE_SUCCESS) {
tscError("%p malloc failed for new queryinfo", pSql);
tscFreeSqlObj(pNew);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
SQueryInfo *pNewQueryInfo = tscGetQueryInfo(&pNew->cmd);
pNewQueryInfo->pUdfInfo = taosArrayInit(4, sizeof(SUdfInfo));
for(int32_t i = 0; i < taosArrayGetSize(pQueryInfo->pUdfInfo); ++i) {
SUdfInfo info = {0};
SUdfInfo* p1 = taosArrayGet(pQueryInfo->pUdfInfo, i);
info = *p1;
info.name = strdup(p1->name);
taosArrayPush(pNewQueryInfo->pUdfInfo, &info);
}
pNew->cmd.active = pNewQueryInfo;
if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) {
tscError("%p malloc failed for payload to get table meta", pSql);
tscFreeSqlObj(pNew);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
tscDebug("%p new pSqlObj:%p to retrieve udf", pSql, pNew);
registerSqlObj(pNew);
pNew->fp = tscTableMetaCallBack;
pNew->param = (void *)pSql->self;
tscDebug("%p metaRid from %" PRId64 " to %" PRId64 , pSql, pSql->metaRid, pNew->self);
pSql->metaRid = pNew->self;
int32_t code = tscBuildAndSendRequest(pNew, NULL);
if (code == TSDB_CODE_SUCCESS) {
code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify application that current process needs to be terminated
}
return code;
}
/**
* retrieve table meta from mnode, and then update the local table meta hashmap.
* @param pSql sql object
@ -2745,6 +2977,7 @@ void tscInitMsgsFp() {
tscBuildMsg[TSDB_SQL_CREATE_DB] = tscBuildCreateDbMsg;
tscBuildMsg[TSDB_SQL_CREATE_USER] = tscBuildUserMsg;
tscBuildMsg[TSDB_SQL_CREATE_FUNCTION] = tscBuildCreateFuncMsg;
tscBuildMsg[TSDB_SQL_CREATE_ACCT] = tscBuildAcctMsg;
tscBuildMsg[TSDB_SQL_ALTER_ACCT] = tscBuildAcctMsg;
@ -2753,6 +2986,7 @@ void tscInitMsgsFp() {
tscBuildMsg[TSDB_SQL_DROP_USER] = tscBuildDropUserAcctMsg;
tscBuildMsg[TSDB_SQL_DROP_ACCT] = tscBuildDropUserAcctMsg;
tscBuildMsg[TSDB_SQL_DROP_DB] = tscBuildDropDbMsg;
tscBuildMsg[TSDB_SQL_DROP_FUNCTION] = tscBuildDropFuncMsg;
tscBuildMsg[TSDB_SQL_SYNC_DB_REPLICA] = tscBuildSyncDbReplicaMsg;
tscBuildMsg[TSDB_SQL_DROP_TABLE] = tscBuildDropTableMsg;
tscBuildMsg[TSDB_SQL_ALTER_USER] = tscBuildUserMsg;
@ -2768,6 +3002,7 @@ void tscInitMsgsFp() {
tscBuildMsg[TSDB_SQL_USE_DB] = tscBuildUseDbMsg;
// tscBuildMsg[TSDB_SQL_META] = tscBuildTableMetaMsg;
tscBuildMsg[TSDB_SQL_STABLEVGROUP] = tscBuildSTableVgroupMsg;
tscBuildMsg[TSDB_SQL_RETRIEVE_FUNC] = tscBuildRetrieveFuncMsg;
tscBuildMsg[TSDB_SQL_HB] = tscBuildHeartBeatMsg;
tscBuildMsg[TSDB_SQL_SHOW] = tscBuildShowMsg;
@ -2786,6 +3021,7 @@ void tscInitMsgsFp() {
tscProcessMsgRsp[TSDB_SQL_META] = tscProcessTableMetaRsp;
tscProcessMsgRsp[TSDB_SQL_STABLEVGROUP] = tscProcessSTableVgroupRsp;
tscProcessMsgRsp[TSDB_SQL_MULTI_META] = tscProcessMultiTableMetaRsp;
tscProcessMsgRsp[TSDB_SQL_RETRIEVE_FUNC] = tscProcessRetrieveFuncRsp;
tscProcessMsgRsp[TSDB_SQL_SHOW] = tscProcessShowRsp;
tscProcessMsgRsp[TSDB_SQL_RETRIEVE] = tscProcessRetrieveRspFromNode; // rsp handled by same function.
@ -2815,3 +3051,4 @@ void tscInitMsgsFp() {
tscKeepConn[TSDB_SQL_FETCH] = 1;
tscKeepConn[TSDB_SQL_HB] = 1;
}

8
src/client/src/tscSql.c
View File

@ -566,7 +566,7 @@ static bool tscKillQueryInDnode(SSqlObj* pSql) {
SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd);
if ((pQueryInfo == NULL) || tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
if ((pQueryInfo == NULL) || pQueryInfo->globalMerge) {
return true;
}
@ -679,7 +679,7 @@ static void tscKillSTableQuery(SSqlObj *pSql) {
SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd);
if (!tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
if (!pQueryInfo->globalMerge) {
return;
}
@ -730,7 +730,7 @@ void taos_stop_query(TAOS_RES *res) {
SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd);
if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
if (pQueryInfo->globalMerge) {
assert(pSql->rpcRid <= 0);
tscKillSTableQuery(pSql);
} else {
@ -979,7 +979,7 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
registerSqlObj(pSql);
tscDebug("0x%"PRIx64" load multiple table meta, tableNameList: %s pObj:%p", pSql->self, tableNameList, pObj);
code = getMultiTableMetaFromMnode(pSql, plist, vgroupList, loadMultiTableMetaCallback);
code = getMultiTableMetaFromMnode(pSql, plist, vgroupList, NULL, loadMultiTableMetaCallback);
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
code = TSDB_CODE_SUCCESS;
}

41
src/client/src/tscSubquery.c
View File

@ -23,6 +23,7 @@
#include "tscSubquery.h"
#include "qTableMeta.h"
#include "tsclient.h"
#include "qUdf.h"
#include "qUtil.h"
#include "qPlan.h"
@ -32,7 +33,7 @@ typedef struct SInsertSupporter {
} SInsertSupporter;
static void freeJoinSubqueryObj(SSqlObj* pSql);
static bool tscHasRemainDataInSubqueryResultSet(SSqlObj *pSql);
//static bool tscHasRemainDataInSubqueryResultSet(SSqlObj *pSql);
static int32_t tsCompare(int32_t order, int64_t left, int64_t right) {
if (left == right) {
@ -1896,7 +1897,7 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
int16_t type = 0;
int32_t inter = 0;
getResultDataInfo(s->type, s->bytes, TSDB_FUNC_TID_TAG, 0, &type, &bytes, &inter, 0, 0);
getResultDataInfo(s->type, s->bytes, TSDB_FUNC_TID_TAG, 0, &type, &bytes, &inter, 0, 0, NULL);
SSchema s1 = {.colId = s->colId, .type = (uint8_t)type, .bytes = bytes};
pSupporter->tagSize = s1.bytes;
@ -2801,6 +2802,28 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p
tscFreeRetrieveSup(pSql);
// set the command flag must be after the semaphore been correctly set.
if (pParentSql->cmd.command != TSDB_SQL_RETRIEVE_EMPTY_RESULT) {
pParentSql->cmd.command = TSDB_SQL_RETRIEVE_GLOBALMERGE;
SQueryInfo *pQueryInfo2 = tscGetQueryInfo(&pParentSql->cmd);
size_t size = tscNumOfExprs(pQueryInfo);
for (int32_t j = 0; j < size; ++j) {
SExprInfo* pExprInfo = tscExprGet(pQueryInfo2, j);
int32_t functionId = pExprInfo->base.functionId;
if (functionId < 0) {
SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo2->pUdfInfo, -1 * functionId - 1);
code = initUdfInfo(pUdfInfo);
if (code != TSDB_CODE_SUCCESS) {
pParentSql->res.code = code;
tscAsyncResultOnError(pParentSql);
}
}
}
}
if (pParentSql->res.code == TSDB_CODE_SUCCESS) {
(*pParentSql->fp)(pParentSql->param, pParentSql, 0);
} else {
@ -3469,20 +3492,20 @@ static UNUSED_FUNC bool tscHasRemainDataInSubqueryResultSet(SSqlObj *pSql) {
SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd);
if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
bool allSubqueryExhausted = true;
for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) {
if (pSql->pSubs[i] == NULL) {
continue;
}
SSqlRes *pRes1 = &pSql->pSubs[i]->res;
SSqlCmd *pCmd1 = &pSql->pSubs[i]->cmd;
SQueryInfo *pQueryInfo1 = tscGetQueryInfo(pCmd1);
assert(pQueryInfo1->numOfTables == 1);
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo1, 0);
/*
* if the global limitation is not reached, and current result has not exhausted, or next more vnodes are
* available, goes on
@ -3493,14 +3516,14 @@ static UNUSED_FUNC bool tscHasRemainDataInSubqueryResultSet(SSqlObj *pSql) {
break;
}
}
hasData = !allSubqueryExhausted;
} else { // otherwise, in case inner join, if any subquery exhausted, query completed.
for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) {
if (pSql->pSubs[i] == 0) {
continue;
}
SSqlRes * pRes1 = &pSql->pSubs[i]->res;
SQueryInfo *pQueryInfo1 = tscGetQueryInfo(&pSql->pSubs[i]->cmd);

7
src/client/src/tscSystem.c
View File

@ -28,6 +28,7 @@
#include "tconfig.h"
#include "ttimezone.h"
#include "tlocale.h"
#include "qScript.h"
// global, not configurable
#define TSC_VAR_NOT_RELEASE 1
@ -148,6 +149,8 @@ void taos_init_imp(void) {
taosInitNotes();
rpcInit();
scriptEnvPoolInit();
tscDebug("starting to initialize TAOS client ...");
tscDebug("Local End Point is:%s", tsLocalEp);
}
@ -202,7 +205,9 @@ void taos_cleanup(void) {
if (atomic_val_compare_exchange_32(&sentinel, TSC_VAR_NOT_RELEASE, TSC_VAR_RELEASED) != TSC_VAR_NOT_RELEASE) {
return;
}
if (tscEmbedded == 0) {
scriptEnvPoolCleanup();
}
taosHashCleanup(tscTableMetaInfo);
tscTableMetaInfo = NULL;

138
src/client/src/tscUtil.c
View File

@ -34,54 +34,54 @@ static void freeQueryInfoImpl(SQueryInfo* pQueryInfo);
int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len) {
int32_t n = 0;
switch (type) {
case TSDB_DATA_TYPE_NULL:
n = sprintf(str, "null");
break;
case TSDB_DATA_TYPE_BOOL:
n = sprintf(str, (*(int8_t*)buf) ? "true" : "false");
break;
case TSDB_DATA_TYPE_TINYINT:
n = sprintf(str, "%d", *(int8_t*)buf);
break;
case TSDB_DATA_TYPE_SMALLINT:
n = sprintf(str, "%d", *(int16_t*)buf);
break;
case TSDB_DATA_TYPE_INT:
n = sprintf(str, "%d", *(int32_t*)buf);
break;
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_TIMESTAMP:
n = sprintf(str, "%" PRId64, *(int64_t*)buf);
break;
case TSDB_DATA_TYPE_FLOAT:
n = sprintf(str, "%f", GET_FLOAT_VAL(buf));
break;
case TSDB_DATA_TYPE_DOUBLE:
n = sprintf(str, "%f", GET_DOUBLE_VAL(buf));
break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
if (bufSize < 0) {
tscError("invalid buf size");
return TSDB_CODE_TSC_INVALID_VALUE;
}
*str = '"';
memcpy(str + 1, buf, bufSize);
*(str + bufSize + 1) = '"';
n = bufSize + 2;
break;
default:
tscError("unsupported type:%d", type);
return TSDB_CODE_TSC_INVALID_VALUE;
@ -216,6 +216,15 @@ bool tscIsProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex) {
for (int32_t i = 0; i < numOfExprs; ++i) {
int32_t functionId = tscExprGet(pQueryInfo, i)->base.functionId;
if (functionId < 0) {
SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1);
if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) {
return false;
}
continue;
}
if (functionId != TSDB_FUNC_PRJ &&
functionId != TSDB_FUNC_TAGPRJ &&
functionId != TSDB_FUNC_TAG &&
@ -266,6 +275,16 @@ bool tscIsProjectionQuery(SQueryInfo* pQueryInfo) {
f != TSDB_FUNC_DERIVATIVE) {
return false;
}
if (f < 0) {
SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * f - 1);
if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) {
return false;
}
continue;
}
}
return true;
@ -338,6 +357,10 @@ bool tsIsArithmeticQueryOnAggResult(SQueryInfo* pQueryInfo) {
}
}
if (tscIsProjectionQuery(pQueryInfo)) {
return false;
}
return false;
}
@ -526,6 +549,15 @@ bool isSimpleAggregateRv(SQueryInfo* pQueryInfo) {
}
int32_t functionId = pExpr->base.functionId;
if (functionId < 0) {
SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1);
if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) {
return true;
}
continue;
}
if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY) {
continue;
}
@ -1186,7 +1218,7 @@ void handleDownstreamOperator(SSqlObj** pSqlObjList, int32_t numOfUpstream, SQue
SOperatorInfo* pSourceOperator = createDummyInputOperator(pSqlObjList[0], pSchema, numOfCol1, pFilterInfo, numOfFilterCols);
pOutput->precision = pSqlObjList[0]->res.precision;
SSchema* schema = NULL;
if (px->numOfTables > 1) {
SOperatorInfo** p = calloc(px->numOfTables, POINTER_BYTES);
@ -1302,6 +1334,12 @@ void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeMeta) {
tfree(pUpQueryInfo);
}
if (pQueryInfo->udfCopy) {
pQueryInfo->pUdfInfo = taosArrayDestroy(pQueryInfo->pUdfInfo);
} else {
pQueryInfo->pUdfInfo = tscDestroyUdfArrayList(pQueryInfo->pUdfInfo);
}
freeQueryInfoImpl(pQueryInfo);
clearAllTableMetaInfo(pQueryInfo, removeMeta);
@ -1537,6 +1575,47 @@ void* tscDestroyBlockArrayList(SArray* pDataBlockList) {
return NULL;
}
void freeUdfInfo(SUdfInfo* pUdfInfo) {
if (pUdfInfo == NULL) {
return;
}
if (pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY]) {
(*(udfDestroyFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY])(&pUdfInfo->init);
}
tfree(pUdfInfo->name);
if (pUdfInfo->path) {
unlink(pUdfInfo->path);
}
tfree(pUdfInfo->path);
tfree(pUdfInfo->content);
taosCloseDll(pUdfInfo->handle);
}
void* tscDestroyUdfArrayList(SArray* pUdfList) {
if (pUdfList == NULL) {
return NULL;
}
size_t size = taosArrayGetSize(pUdfList);
for (int32_t i = 0; i < size; i++) {
SUdfInfo* udf = taosArrayGet(pUdfList, i);
freeUdfInfo(udf);
}
taosArrayDestroy(pUdfList);
return NULL;
}
void* tscDestroyBlockHashTable(SHashObj* pBlockHashTable, bool removeMeta) {
if (pBlockHashTable == NULL) {
return NULL;
@ -2237,7 +2316,7 @@ void tscFieldInfoCopy(SFieldInfo* pFieldInfo, const SFieldInfo* pSrc, const SArr
assert(pfield->pExpr->pExpr != NULL);
p.pExpr = calloc(1, sizeof(SExprInfo));
tscExprAssign(p.pExpr, pfield->pExpr);
}
}
taosArrayPush(pFieldInfo->internalField, &p);
}
@ -3098,7 +3177,7 @@ int32_t tscQueryInfoCopy(SQueryInfo* pQueryInfo, const SQueryInfo* pSrc) {
pQueryInfo->fillVal = NULL;
pQueryInfo->numOfFillVal = 0;;
pQueryInfo->clauseLimit = pSrc->clauseLimit;
pQueryInfo->prjOffset = pSrc->prjOffset;
pQueryInfo->prjOffset = pSrc->prjOffset;
pQueryInfo->numOfTables = 0;
pQueryInfo->window = pSrc->window;
pQueryInfo->sessionWindow = pSrc->sessionWindow;
@ -3169,6 +3248,14 @@ int32_t tscQueryInfoCopy(SQueryInfo* pQueryInfo, const SQueryInfo* pSrc) {
tscAddTableMetaInfo(pQueryInfo, &p1->name, pMeta, p1->vgroupList, p1->tagColList, p1->pVgroupTables);
}
SArray *pUdfInfo = NULL;
if (pSrc->pUdfInfo) {
pUdfInfo = taosArrayDup(pSrc->pUdfInfo);
}
pQueryInfo->pUdfInfo = pUdfInfo;
pQueryInfo->udfCopy = true;
_error:
return code;
}
@ -3215,7 +3302,9 @@ void tscVgroupTableCopy(SVgroupTableInfo* info, SVgroupTableInfo* pInfo) {
info->vgInfo.epAddr[j].fqdn = strdup(pInfo->vgInfo.epAddr[j].fqdn);
}
info->itemList = taosArrayDup(pInfo->itemList);
if (pInfo->itemList) {
info->itemList = taosArrayDup(pInfo->itemList);
}
}
SArray* tscVgroupTableInfoDup(SArray* pVgroupTables) {
@ -3466,6 +3555,11 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t
SQueryInfo* pNewQueryInfo = tscGetQueryInfo(pnCmd);
if (pQueryInfo->pUdfInfo) {
pNewQueryInfo->pUdfInfo = taosArrayDup(pQueryInfo->pUdfInfo);
pNewQueryInfo->udfCopy = true;
}
pNewQueryInfo->command = pQueryInfo->command;
pnCmd->active = pNewQueryInfo;
@ -3485,6 +3579,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t
pNewQueryInfo->numOfTables = 0;
pNewQueryInfo->pTableMetaInfo = NULL;
pNewQueryInfo->bufLen = pQueryInfo->bufLen;
pNewQueryInfo->buf = malloc(pQueryInfo->bufLen);
if (pNewQueryInfo->buf == NULL) {
terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
@ -4327,7 +4422,7 @@ int32_t createProjectionExpr(SQueryInfo* pQueryInfo, STableMetaInfo* pTableMetaI
int32_t inter = 0;
getResultDataInfo(pSource->base.colType, pSource->base.colBytes, functionId, 0, &pse->resType,
&pse->resBytes, &inter, 0, false);
&pse->resBytes, &inter, 0, false, NULL);
pse->colType = pse->resType;
pse->colBytes = pse->resBytes;
@ -4394,8 +4489,14 @@ static int32_t createGlobalAggregateExpr(SQueryAttr* pQueryAttr, SQueryInfo* pQu
functionId = TSDB_FUNC_STDDEV;
}
SUdfInfo* pUdfInfo = NULL;
if (functionId < 0) {
pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1);
}
getResultDataInfo(pExpr->base.colType, pExpr->base.colBytes, functionId, 0, &pse->resType, &pse->resBytes, &inter,
0, false);
0, false, pUdfInfo);
}
}
@ -4471,7 +4572,8 @@ int32_t tscCreateQueryFromQueryInfo(SQueryInfo* pQueryInfo, SQueryAttr* pQueryAt
pQueryAttr->order = pQueryInfo->order;
pQueryAttr->fillType = pQueryInfo->fillType;
pQueryAttr->havingNum = pQueryInfo->havingFieldNum;
pQueryAttr->pUdfInfo = pQueryInfo->pUdfInfo;
if (pQueryInfo->order.order == TSDB_ORDER_ASC) { // TODO refactor
pQueryAttr->window = pQueryInfo->window;
} else {

3
src/common/inc/tcmdtype.h
View File

@ -41,8 +41,10 @@ enum {
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_MGMT, "mgmt" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_DB, "create-db" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_TABLE, "create-table" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_FUNCTION, "create-function" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_DB, "drop-db" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_TABLE, "drop-table" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_FUNCTION, "drop-function" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_ACCT, "create-acct" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_USER, "create-user" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_ACCT, "drop-acct" )
@ -74,6 +76,7 @@ enum {
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_STABLEVGROUP, "stable-vgroup" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_MULTI_META, "multi-meta" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_HB, "heart-beat" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_RETRIEVE_FUNC, "retrieve-function" )
// SQL below for client local
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_LOCAL, "local" )

8
src/common/inc/tdataformat.h
View File

@ -58,6 +58,14 @@ extern const SNCharNullT NcharNull;
memcpy(varDataVal(x), (str), __len); \
} while (0);
#define STR_TO_NET_VARSTR(x, str) \
do { \
VarDataLenT __len = (VarDataLenT)strlen(str); \
*(VarDataLenT *)(x) = htons(__len); \
memcpy(varDataVal(x), (str), __len); \
} while (0);
#define STR_WITH_MAXSIZE_TO_VARSTR(x, str, _maxs) \
do { \
char *_e = stpncpy(varDataVal(x), (str), (_maxs)-VARSTR_HEADER_SIZE); \

2
src/connector/go

@ -1 +1 @@
Subproject commit b8f76da4a708d158ec3cc4b844571dc4414e36b4
Subproject commit 7a26c432f8b4203e42344ff3290b9b9b01b983d5

2
src/connector/grafanaplugin

@ -1 +1 @@
Subproject commit 3530c6df097134a410bacec6b3cd013ef38a61aa
Subproject commit 23cf0b933510af5e87b61bace0f6ff2f06d8eaac

2
src/connector/hivemq-tdengine-extension

@ -1 +1 @@
Subproject commit ce5201014136503d34fecbd56494b67b4961056c
Subproject commit b62a26ecc164a310104df57691691b237e091c89

2
src/dnode/CMakeLists.txt
View File

@ -18,7 +18,7 @@ ELSE ()
ENDIF ()
ADD_EXECUTABLE(taosd ${SRC})
TARGET_LINK_LIBRARIES(taosd mnode monitor http tsdb twal vnode cJson lz4 balance sync ${LINK_JEMALLOC})
TARGET_LINK_LIBRARIES(taosd mnode monitor http tsdb twal vnode cJson lua lz4 balance sync ${LINK_JEMALLOC})
IF (TD_SOMODE_STATIC)
TARGET_LINK_LIBRARIES(taosd taos_static)

4
src/dnode/src/dnodeMain.c
View File

@ -40,6 +40,7 @@
#include "dnodeShell.h"
#include "dnodeTelemetry.h"
#include "module.h"
#include "qScript.h"
#include "mnode.h"
#if !defined(_MODULE) || !defined(_TD_LINUX)
@ -84,6 +85,7 @@ static SStep tsDnodeSteps[] = {
{"dnode-shell", dnodeInitShell, dnodeCleanupShell},
{"dnode-statustmr", dnodeInitStatusTimer,dnodeCleanupStatusTimer},
{"dnode-telemetry", dnodeInitTelemetry, dnodeCleanupTelemetry},
{"dnode-script", scriptEnvPoolInit, scriptEnvPoolCleanup},
};
static SStep tsDnodeCompactSteps[] = {
@ -266,7 +268,7 @@ static int32_t dnodeInitStorage() {
return -1;
}
}
//TODO(dengyihao): no need to init here
//TODO(dengyihao): no need to init here
if (dnodeCreateDir(tsMnodeDir) < 0) {
dError("failed to create dir: %s, reason: %s", tsMnodeDir, strerror(errno));
return -1;

3
src/dnode/src/dnodeShell.c
View File

@ -48,9 +48,11 @@ int32_t dnodeInitShell() {
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DNODE] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DB] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TP] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_FUNCTION] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DB] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_SYNC_DB] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_TP] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_FUNCTION] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_DB] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TP] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TABLE]= dnodeDispatchToMWriteQueue;
@ -72,6 +74,7 @@ int32_t dnodeInitShell() {
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLES_META] = dnodeDispatchToMReadQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_SHOW] = dnodeDispatchToMReadQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE] = dnodeDispatchToMReadQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE_FUNC] = dnodeDispatchToMReadQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_NETWORK_TEST] = dnodeSendStartupStep;

2
src/inc/query.h
View File

@ -28,7 +28,7 @@ typedef void* qinfo_t;
* @param qinfo
* @return
*/
int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMsg, qinfo_t* qinfo, uint64_t *qId);
int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMsg, qinfo_t* qinfo, uint64_t qId);
/**

10
src/inc/taosdef.h
View File

@ -186,6 +186,10 @@ do { \
#define TSDB_NODE_NAME_LEN 64
#define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string
#define TSDB_DB_NAME_LEN 33
#define TSDB_FUNC_NAME_LEN 65
#define TSDB_FUNC_CODE_LEN (65535 - 512)
#define TSDB_FUNC_BUF_SIZE 512
#define TSDB_TYPE_STR_MAX_LEN 32
#define TSDB_TABLE_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_TABLE_NAME_LEN)
#define TSDB_COL_NAME_LEN 65
#define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64
@ -197,7 +201,7 @@ do { \
/**
* In some scenarios uint16_t (0~65535) is used to store the row len.
* - Firstly, we use 65531(65535 - 4), as the SDataRow and SKVRow including 4 bits header.
* - Firstly, we use 65531(65535 - 4), as the SDataRow/SKVRow contains 4 bits header.
* - Secondly, if all cols are VarDataT type except primary key, we need 4 bits to store the offset, thus
* the final value is 65531-(4096-1)*4 = 49151.
*/
@ -339,6 +343,10 @@ do { \
#define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type
#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode
#define TSDB_UDF_TYPE_SCALAR 1
#define TSDB_UDF_TYPE_AGGREGATE 2
/*
* 1. ordinary sub query for select * from super_table
* 2. all sqlobj generated by createSubqueryObj with this flag

9
src/inc/taoserror.h
View File

@ -100,6 +100,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_TSC_DB_NOT_SELECTED TAOS_DEF_ERROR_CODE(0, 0x0217) //"Database not specified or available")
#define TSDB_CODE_TSC_INVALID_TABLE_NAME TAOS_DEF_ERROR_CODE(0, 0x0218) //"Table does not exist")
#define TSDB_CODE_TSC_EXCEED_SQL_LIMIT TAOS_DEF_ERROR_CODE(0, 0x0219) //"SQL statement too long check maxSQLLength config")
#define TSDB_CODE_TSC_FILE_EMPTY TAOS_DEF_ERROR_CODE(0, 0x021A) //"File is empty")
// mnode
#define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0300) //"Message not processed")
@ -174,6 +175,13 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_INVALID_STABLE_NAME TAOS_DEF_ERROR_CODE(0, 0x036D) //"Super table does not exist")
#define TSDB_CODE_MND_INVALID_CREATE_TABLE_MSG TAOS_DEF_ERROR_CODE(0, 0x036E) //"Invalid create table message")
#define TSDB_CODE_MND_INVALID_FUNC_NAME TAOS_DEF_ERROR_CODE(0, 0x0370) //"Invalid func name")
#define TSDB_CODE_MND_INVALID_FUNC_LEN TAOS_DEF_ERROR_CODE(0, 0x0371) //"Invalid func length")
#define TSDB_CODE_MND_INVALID_FUNC_CODE TAOS_DEF_ERROR_CODE(0, 0x0372) //"Invalid func code")
#define TSDB_CODE_MND_FUNC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0373) //"Func already exists")
#define TSDB_CODE_MND_INVALID_FUNC TAOS_DEF_ERROR_CODE(0, 0x0374) //"Invalid func")
#define TSDB_CODE_MND_INVALID_FUNC_BUFSIZE TAOS_DEF_ERROR_CODE(0, 0x0375) //"Invalid func bufSize")
#define TSDB_CODE_MND_DB_NOT_SELECTED TAOS_DEF_ERROR_CODE(0, 0x0380) //"Database not specified or available")
#define TSDB_CODE_MND_DB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0381) //"Database already exists")
#define TSDB_CODE_MND_INVALID_DB_OPTION TAOS_DEF_ERROR_CODE(0, 0x0382) //"Invalid database options")
@ -261,6 +269,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW TAOS_DEF_ERROR_CODE(0, 0x070A) //"Too many time window in query")
#define TSDB_CODE_QRY_NOT_ENOUGH_BUFFER TAOS_DEF_ERROR_CODE(0, 0x070B) //"Query buffer limit has reached")
#define TSDB_CODE_QRY_INCONSISTAN TAOS_DEF_ERROR_CODE(0, 0x070C) //"File inconsistency in replica")
#define TSDB_CODE_QRY_SYS_ERROR TAOS_DEF_ERROR_CODE(0, 0x070D) //"System error")
// grant

61
src/inc/taosmsg.h
View File

@ -77,8 +77,10 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_DROP_USER, "drop-user" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_CREATE_DNODE, "create-dnode" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_DROP_DNODE, "drop-dnode" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_CREATE_DB, "create-db" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_DROP_DB, "drop-db" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_USE_DB, "use-db" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_CREATE_FUNCTION, "create-function" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_DROP_DB, "drop-db" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_DROP_FUNCTION, "drop-function" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_USE_DB, "use-db" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_ALTER_DB, "alter-db" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_SYNC_DB, "sync-db-replica" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_CREATE_TABLE, "create-table" )
@ -96,7 +98,7 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_KILL_STREAM, "kill-stream" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_KILL_CONN, "kill-conn" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_CONFIG_DNODE, "cm-config-dnode" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_HEARTBEAT, "heartbeat" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY8, "dummy8" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_RETRIEVE_FUNC, "retrieve-func" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY9, "dummy9" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY10, "dummy10" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY11, "dummy11" )
@ -153,6 +155,7 @@ enum _mgmt_table {
TSDB_MGMT_TABLE_STREAMTABLES,
TSDB_MGMT_TABLE_CLUSTER,
TSDB_MGMT_TABLE_TP,
TSDB_MGMT_TABLE_FUNCTION,
TSDB_MGMT_TABLE_MAX,
};
@ -507,6 +510,9 @@ typedef struct {
int32_t prevResultLen; // previous result length
int32_t numOfOperator;
int32_t tableScanOperator;// table scan operator. -1 means no scan operator
int32_t udfNum; // number of udf function
int32_t udfContentOffset;
int32_t udfContentLen;
SColumnInfo tableCols[];
} SQueryTableMsg;
@ -571,6 +577,41 @@ typedef struct {
int8_t reserve[5];
} SCreateDbMsg, SAlterDbMsg;
typedef struct {
char name[TSDB_FUNC_NAME_LEN];
char path[PATH_MAX];
int32_t funcType;
uint8_t outputType;
int16_t outputLen;
int32_t bufSize;
int32_t codeLen;
char code[];
} SCreateFuncMsg;
typedef struct {
int32_t num;
char name[];
} SRetrieveFuncMsg;
typedef struct {
char name[TSDB_FUNC_NAME_LEN];
int32_t funcType;
int8_t resType;
int16_t resBytes;
int32_t bufSize;
int32_t len;
char content[];
} SFunctionInfoMsg;
typedef struct {
int32_t num;
char content[];
} SUdfFuncMsg;
typedef struct {
char name[TSDB_FUNC_NAME_LEN];
} SDropFuncMsg;
typedef struct {
char db[TSDB_TABLE_FNAME_LEN];
uint8_t ignoreNotExists;
@ -712,6 +753,7 @@ typedef struct {
typedef struct {
int32_t numOfVgroups;
int32_t numOfTables;
int32_t numOfUdfs;
char tableNames[];
} SMultiTableInfoMsg;
@ -760,12 +802,13 @@ typedef struct STableMetaMsg {
} STableMetaMsg;
typedef struct SMultiTableMeta {
int32_t numOfTables;
int32_t numOfVgroup;
uint32_t contLen:31;
uint8_t compressed:1; // denote if compressed or not
uint32_t rawLen; // size before compress
char meta[];
int32_t numOfTables;
int32_t numOfVgroup;
int32_t numOfUdf;
int32_t contLen;
uint8_t compressed; // denote if compressed or not
uint32_t rawLen; // size before compress
char meta[];
} SMultiTableMeta;
typedef struct {

291
src/inc/ttokendef.h
View File

@ -62,149 +62,154 @@
#define TK_SHOW 44
#define TK_DATABASES 45
#define TK_TOPICS 46
#define TK_MNODES 47
#define TK_DNODES 48
#define TK_ACCOUNTS 49
#define TK_USERS 50
#define TK_MODULES 51
#define TK_QUERIES 52
#define TK_CONNECTIONS 53
#define TK_STREAMS 54
#define TK_VARIABLES 55
#define TK_SCORES 56
#define TK_GRANTS 57
#define TK_VNODES 58
#define TK_IPTOKEN 59
#define TK_DOT 60
#define TK_CREATE 61
#define TK_TABLE 62
#define TK_STABLE 63
#define TK_DATABASE 64
#define TK_TABLES 65
#define TK_STABLES 66
#define TK_VGROUPS 67
#define TK_DROP 68
#define TK_TOPIC 69
#define TK_DNODE 70
#define TK_USER 71
#define TK_ACCOUNT 72
#define TK_USE 73
#define TK_DESCRIBE 74
#define TK_ALTER 75
#define TK_PASS 76
#define TK_PRIVILEGE 77
#define TK_LOCAL 78
#define TK_COMPACT 79
#define TK_LP 80
#define TK_RP 81
#define TK_IF 82
#define TK_EXISTS 83
#define TK_PPS 84
#define TK_TSERIES 85
#define TK_DBS 86
#define TK_STORAGE 87
#define TK_QTIME 88
#define TK_CONNS 89
#define TK_STATE 90
#define TK_COMMA 91
#define TK_KEEP 92
#define TK_CACHE 93
#define TK_REPLICA 94
#define TK_QUORUM 95
#define TK_DAYS 96
#define TK_MINROWS 97
#define TK_MAXROWS 98
#define TK_BLOCKS 99
#define TK_CTIME 100
#define TK_WAL 101
#define TK_FSYNC 102
#define TK_COMP 103
#define TK_PRECISION 104
#define TK_UPDATE 105
#define TK_CACHELAST 106
#define TK_PARTITIONS 107
#define TK_UNSIGNED 108
#define TK_TAGS 109
#define TK_USING 110
#define TK_AS 111
#define TK_NULL 112
#define TK_NOW 113
#define TK_SELECT 114
#define TK_UNION 115
#define TK_ALL 116
#define TK_DISTINCT 117
#define TK_FROM 118
#define TK_VARIABLE 119
#define TK_INTERVAL 120
#define TK_SESSION 121
#define TK_STATE_WINDOW 122
#define TK_FILL 123
#define TK_SLIDING 124
#define TK_ORDER 125
#define TK_BY 126
#define TK_ASC 127
#define TK_DESC 128
#define TK_GROUP 129
#define TK_HAVING 130
#define TK_LIMIT 131
#define TK_OFFSET 132
#define TK_SLIMIT 133
#define TK_SOFFSET 134
#define TK_WHERE 135
#define TK_RESET 136
#define TK_QUERY 137
#define TK_SYNCDB 138
#define TK_ADD 139
#define TK_COLUMN 140
#define TK_MODIFY 141
#define TK_TAG 142
#define TK_CHANGE 143
#define TK_SET 144
#define TK_KILL 145
#define TK_CONNECTION 146
#define TK_STREAM 147
#define TK_COLON 148
#define TK_ABORT 149
#define TK_AFTER 150
#define TK_ATTACH 151
#define TK_BEFORE 152
#define TK_BEGIN 153
#define TK_CASCADE 154
#define TK_CLUSTER 155
#define TK_CONFLICT 156
#define TK_COPY 157
#define TK_DEFERRED 158
#define TK_DELIMITERS 159
#define TK_DETACH 160
#define TK_EACH 161
#define TK_END 162
#define TK_EXPLAIN 163
#define TK_FAIL 164
#define TK_FOR 165
#define TK_IGNORE 166
#define TK_IMMEDIATE 167
#define TK_INITIALLY 168
#define TK_INSTEAD 169
#define TK_MATCH 170
#define TK_KEY 171
#define TK_OF 172
#define TK_RAISE 173
#define TK_REPLACE 174
#define TK_RESTRICT 175
#define TK_ROW 176
#define TK_STATEMENT 177
#define TK_TRIGGER 178
#define TK_VIEW 179
#define TK_SEMI 180
#define TK_NONE 181
#define TK_PREV 182
#define TK_LINEAR 183
#define TK_IMPORT 184
#define TK_TBNAME 185
#define TK_JOIN 186
#define TK_INSERT 187
#define TK_INTO 188
#define TK_VALUES 189
#define TK_FUNCTIONS 47
#define TK_MNODES 48
#define TK_DNODES 49
#define TK_ACCOUNTS 50
#define TK_USERS 51
#define TK_MODULES 52
#define TK_QUERIES 53
#define TK_CONNECTIONS 54
#define TK_STREAMS 55
#define TK_VARIABLES 56
#define TK_SCORES 57
#define TK_GRANTS 58
#define TK_VNODES 59
#define TK_IPTOKEN 60
#define TK_DOT 61
#define TK_CREATE 62
#define TK_TABLE 63
#define TK_STABLE 64
#define TK_DATABASE 65
#define TK_TABLES 66
#define TK_STABLES 67
#define TK_VGROUPS 68
#define TK_DROP 69
#define TK_TOPIC 70
#define TK_FUNCTION 71
#define TK_DNODE 72
#define TK_USER 73
#define TK_ACCOUNT 74
#define TK_USE 75
#define TK_DESCRIBE 76
#define TK_ALTER 77
#define TK_PASS 78
#define TK_PRIVILEGE 79
#define TK_LOCAL 80
#define TK_COMPACT 81
#define TK_LP 82
#define TK_RP 83
#define TK_IF 84
#define TK_EXISTS 85
#define TK_AS 86
#define TK_OUTPUTTYPE 87
#define TK_AGGREGATE 88
#define TK_BUFSIZE 89
#define TK_PPS 90
#define TK_TSERIES 91
#define TK_DBS 92
#define TK_STORAGE 93
#define TK_QTIME 94
#define TK_CONNS 95
#define TK_STATE 96
#define TK_COMMA 97
#define TK_KEEP 98
#define TK_CACHE 99
#define TK_REPLICA 100
#define TK_QUORUM 101
#define TK_DAYS 102
#define TK_MINROWS 103
#define TK_MAXROWS 104
#define TK_BLOCKS 105
#define TK_CTIME 106
#define TK_WAL 107
#define TK_FSYNC 108
#define TK_COMP 109
#define TK_PRECISION 110
#define TK_UPDATE 111
#define TK_CACHELAST 112
#define TK_PARTITIONS 113
#define TK_UNSIGNED 114
#define TK_TAGS 115
#define TK_USING 116
#define TK_NULL 117
#define TK_NOW 118
#define TK_SELECT 119
#define TK_UNION 120
#define TK_ALL 121
#define TK_DISTINCT 122
#define TK_FROM 123
#define TK_VARIABLE 124
#define TK_INTERVAL 125
#define TK_SESSION 126
#define TK_STATE_WINDOW 127
#define TK_FILL 128
#define TK_SLIDING 129
#define TK_ORDER 130
#define TK_BY 131
#define TK_ASC 132
#define TK_DESC 133
#define TK_GROUP 134
#define TK_HAVING 135
#define TK_LIMIT 136
#define TK_OFFSET 137
#define TK_SLIMIT 138
#define TK_SOFFSET 139
#define TK_WHERE 140
#define TK_RESET 141
#define TK_QUERY 142
#define TK_SYNCDB 143
#define TK_ADD 144
#define TK_COLUMN 145
#define TK_MODIFY 146
#define TK_TAG 147
#define TK_CHANGE 148
#define TK_SET 149
#define TK_KILL 150
#define TK_CONNECTION 151
#define TK_STREAM 152
#define TK_COLON 153
#define TK_ABORT 154
#define TK_AFTER 155
#define TK_ATTACH 156
#define TK_BEFORE 157
#define TK_BEGIN 158
#define TK_CASCADE 159
#define TK_CLUSTER 160
#define TK_CONFLICT 161
#define TK_COPY 162
#define TK_DEFERRED 163
#define TK_DELIMITERS 164
#define TK_DETACH 165
#define TK_EACH 166
#define TK_END 167
#define TK_EXPLAIN 168
#define TK_FAIL 169
#define TK_FOR 170
#define TK_IGNORE 171
#define TK_IMMEDIATE 172
#define TK_INITIALLY 173
#define TK_INSTEAD 174
#define TK_MATCH 175
#define TK_KEY 176
#define TK_OF 177
#define TK_RAISE 178
#define TK_REPLACE 179
#define TK_RESTRICT 180
#define TK_ROW 181
#define TK_STATEMENT 182
#define TK_TRIGGER 183
#define TK_VIEW 184
#define TK_SEMI 185
#define TK_NONE 186
#define TK_PREV 187
#define TK_LINEAR 188
#define TK_IMPORT 189
#define TK_TBNAME 190
#define TK_JOIN 191
#define TK_INSERT 192
#define TK_INTO 193
#define TK_VALUES 194
#define TK_SPACE 300

4
src/inc/ttype.h
View File

@ -30,6 +30,10 @@ typedef struct tstr {
#define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT) (_len))
#define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_BINARY) || ((t) == TSDB_DATA_TYPE_NCHAR))
#define varDataNetLen(v) (htons(((VarDataLenT *)(v))[0]))
#define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v))
// this data type is internally used only in 'in' query to hold the values
#define TSDB_DATA_TYPE_ARRAY (1000)

4
src/kit/shell/CMakeLists.txt
View File

@ -19,9 +19,9 @@ ELSE ()
ENDIF ()
IF (TD_SOMODE_STATIC)
TARGET_LINK_LIBRARIES(shell taos_static ${LINK_JEMALLOC})
TARGET_LINK_LIBRARIES(shell taos_static lua ${LINK_JEMALLOC})
ELSE ()
TARGET_LINK_LIBRARIES(shell taos ${LINK_JEMALLOC})
TARGET_LINK_LIBRARIES(shell taos lua ${LINK_JEMALLOC})
ENDIF ()
SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos)

10
src/kit/taosdemo/CMakeLists.txt
View File

@ -67,7 +67,7 @@ IF (TD_LINUX)
ADD_EXECUTABLE(taosdemo ${SRC})
IF (TD_SOMODE_STATIC)
TARGET_LINK_LIBRARIES(taosdemo taos_static cJson ${LINK_JEMALLOC})
TARGET_LINK_LIBRARIES(taosdemo taos_static cJson lua ${LINK_JEMALLOC})
ELSE ()
TARGET_LINK_LIBRARIES(taosdemo taos cJson ${LINK_JEMALLOC})
ENDIF ()
@ -76,9 +76,9 @@ ELSEIF (TD_WINDOWS)
ADD_EXECUTABLE(taosdemo ${SRC})
SET_SOURCE_FILES_PROPERTIES(./taosdemo.c PROPERTIES COMPILE_FLAGS -w)
IF (TD_SOMODE_STATIC)
TARGET_LINK_LIBRARIES(taosdemo taos_static cJson)
TARGET_LINK_LIBRARIES(taosdemo taos_static cJson lua)
ELSE ()
TARGET_LINK_LIBRARIES(taosdemo taos cJson)
TARGET_LINK_LIBRARIES(taosdemo taos cJson lua)
ENDIF ()
ELSEIF (TD_DARWIN)
# missing a few dependencies, such as <argp.h>
@ -86,9 +86,9 @@ ELSEIF (TD_DARWIN)
ADD_EXECUTABLE(taosdemo ${SRC})
IF (TD_SOMODE_STATIC)
TARGET_LINK_LIBRARIES(taosdemo taos_static cJson)
TARGET_LINK_LIBRARIES(taosdemo taos_static cJson lua)
ELSE ()
TARGET_LINK_LIBRARIES(taosdemo taos cJson)
TARGET_LINK_LIBRARIES(taosdemo taos cJson lua)
ENDIF ()
ENDIF ()

204
src/kit/taosdemo/taosdemo.c
View File

@ -106,9 +106,9 @@ enum TEST_MODE {
typedef enum CREATE_SUB_TALBE_MOD_EN {
PRE_CREATE_SUBTBL,
AUTO_CREATE_SUBTBL,
NO_CREATE_SUBTBL
PRE_CREATE_SUBTBL,
AUTO_CREATE_SUBTBL,
NO_CREATE_SUBTBL
} CREATE_SUB_TALBE_MOD_EN;
typedef enum TALBE_EXISTS_EN {
@ -637,7 +637,7 @@ static FILE * g_fpOfInsertResult = NULL;
#define performancePrint(fmt, ...) \
do { if (g_args.performance_print) \
fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0)
fprintf(stderr, "PERF: "fmt, __VA_ARGS__); } while(0)
#define errorPrint(fmt, ...) \
do { fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); } while(0)
@ -3183,8 +3183,10 @@ static void createChildTables() {
if (g_Dbs.db[i].superTblCount > 0) {
// with super table
for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
if ((AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable)
|| (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) {
if ((AUTO_CREATE_SUBTBL
== g_Dbs.db[i].superTbls[j].autoCreateTable)
|| (TBL_ALREADY_EXISTS
== g_Dbs.db[i].superTbls[j].childTblExists)) {
continue;
}
verbosePrint("%s() LN%d: %s\n", __func__, __LINE__,
@ -4156,6 +4158,22 @@ static bool getMetaFromInsertJsonFile(cJSON* root) {
goto PARSE_OVER;
}
*/
cJSON* insertRows = cJSON_GetObjectItem(stbInfo, "insert_rows");
if (insertRows && insertRows->type == cJSON_Number) {
if (insertRows->valueint < 0) {
errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n",
__func__, __LINE__);
goto PARSE_OVER;
}
g_Dbs.db[i].superTbls[j].insertRows = insertRows->valueint;
} else if (!insertRows) {
g_Dbs.db[i].superTbls[j].insertRows = 0x7FFFFFFFFFFFFFFF;
} else {
errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n",
__func__, __LINE__);
goto PARSE_OVER;
}
cJSON* stbInterlaceRows = cJSON_GetObjectItem(stbInfo, "interlace_rows");
if (stbInterlaceRows && stbInterlaceRows->type == cJSON_Number) {
if (stbInterlaceRows->valueint < 0) {
@ -4164,15 +4182,15 @@ static bool getMetaFromInsertJsonFile(cJSON* root) {
goto PARSE_OVER;
}
g_Dbs.db[i].superTbls[j].interlaceRows = stbInterlaceRows->valueint;
// rows per table need be less than insert batch
if (g_Dbs.db[i].superTbls[j].interlaceRows > g_args.num_of_RPR) {
printf("NOTICE: db[%d].superTbl[%d]'s interlace rows value %u > num_of_records_per_req %u\n\n",
i, j, g_Dbs.db[i].superTbls[j].interlaceRows,
g_args.num_of_RPR);
printf(" interlace rows value will be set to num_of_records_per_req %u\n\n",
g_args.num_of_RPR);
prompt();
g_Dbs.db[i].superTbls[j].interlaceRows = g_args.num_of_RPR;
if (g_Dbs.db[i].superTbls[j].interlaceRows > g_Dbs.db[i].superTbls[j].insertRows) {
printf("NOTICE: db[%d].superTbl[%d]'s interlace rows value %u > insert_rows %"PRId64"\n\n",
i, j, g_Dbs.db[i].superTbls[j].interlaceRows,
g_Dbs.db[i].superTbls[j].insertRows);
printf(" interlace rows value will be set to insert_rows %"PRId64"\n\n",
g_Dbs.db[i].superTbls[j].insertRows);
prompt();
g_Dbs.db[i].superTbls[j].interlaceRows = g_Dbs.db[i].superTbls[j].insertRows;
}
} else if (!stbInterlaceRows) {
g_Dbs.db[i].superTbls[j].interlaceRows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req
@ -4209,22 +4227,6 @@ static bool getMetaFromInsertJsonFile(cJSON* root) {
goto PARSE_OVER;
}
cJSON* insertRows = cJSON_GetObjectItem(stbInfo, "insert_rows");
if (insertRows && insertRows->type == cJSON_Number) {
if (insertRows->valueint < 0) {
errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n",
__func__, __LINE__);
goto PARSE_OVER;
}
g_Dbs.db[i].superTbls[j].insertRows = insertRows->valueint;
} else if (!insertRows) {
g_Dbs.db[i].superTbls[j].insertRows = 0x7FFFFFFFFFFFFFFF;
} else {
errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n",
__func__, __LINE__);
goto PARSE_OVER;
}
cJSON* insertInterval = cJSON_GetObjectItem(stbInfo, "insert_interval");
if (insertInterval && insertInterval->type == cJSON_Number) {
g_Dbs.db[i].superTbls[j].insertInterval = insertInterval->valueint;
@ -4981,12 +4983,20 @@ static int64_t generateData(char *recBuf, char **data_type,
bool b = rand_bool() & 1;
pstr += sprintf(pstr, ",%s", b ? "true" : "false");
} else if (strcasecmp(data_type[i % columnCount], "BINARY") == 0) {
char *s = malloc(lenOfBinary);
char *s = malloc(lenOfBinary + 1);
if (s == NULL) {
errorPrint("%s() LN%d, memory allocation %d bytes failed\n",
__func__, __LINE__, lenOfBinary + 1);
}
rand_string(s, lenOfBinary);
pstr += sprintf(pstr, ",\"%s\"", s);
free(s);
} else if (strcasecmp(data_type[i % columnCount], "NCHAR") == 0) {
char *s = malloc(lenOfBinary);
char *s = malloc(lenOfBinary + 1);
if (s == NULL) {
errorPrint("%s() LN%d, memory allocation %d bytes failed\n",
__func__, __LINE__, lenOfBinary + 1);
}
rand_string(s, lenOfBinary);
pstr += sprintf(pstr, ",\"%s\"", s);
free(s);
@ -5143,12 +5153,18 @@ static int32_t generateDataTailWithoutStb(
char **data_type = g_args.datatype;
int lenOfBinary = g_args.len_of_binary;
retLen = generateData(data, data_type,
startTime + getTSRandTail(
(int64_t) DEFAULT_TIMESTAMP_STEP, k,
g_args.disorderRatio,
g_args.disorderRange),
lenOfBinary);
if (g_args.disorderRatio) {
retLen = generateData(data, data_type,
startTime + getTSRandTail(
(int64_t) DEFAULT_TIMESTAMP_STEP, k,
g_args.disorderRatio,
g_args.disorderRange),
lenOfBinary);
} else {
retLen = generateData(data, data_type,
startTime + (int64_t) (DEFAULT_TIMESTAMP_STEP* k),
lenOfBinary);
}
if (len > remainderBufLen)
break;
@ -5200,14 +5216,14 @@ static int32_t generateStbDataTail(
bool tsRand;
if (0 == strncasecmp(superTblInfo->dataSource, "rand", strlen("rand"))) {
tsRand = true;
tsRand = true;
} else {
tsRand = false;
tsRand = false;
}
verbosePrint("%s() LN%d batch=%u buflen=%"PRId64"\n",
__func__, __LINE__, batch, remainderBufLen);
int32_t k = 0;
int32_t k;
for (k = 0; k < batch;) {
char data[MAX_DATA_SIZE];
memset(data, 0, MAX_DATA_SIZE);
@ -5656,10 +5672,15 @@ static int32_t prepareStmtWithoutStb(
bind_ts = (int64_t *)ptr;
bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP;
*bind_ts = startTime + getTSRandTail(
(int64_t)DEFAULT_TIMESTAMP_STEP, k,
g_args.disorderRatio,
g_args.disorderRange);
if (g_args.disorderRatio) {
*bind_ts = startTime + getTSRandTail(
(int64_t)DEFAULT_TIMESTAMP_STEP, k,
g_args.disorderRatio,
g_args.disorderRange);
} else {
*bind_ts = startTime + (int64_t)(DEFAULT_TIMESTAMP_STEP * k);
}
bind->buffer_length = sizeof(int64_t);
bind->buffer = bind_ts;
bind->length = &bind->buffer_length;
@ -5744,7 +5765,7 @@ static int32_t prepareStbStmt(
bind_ts = (int64_t *)ptr;
bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP;
if (sourceRand) {
if (stbInfo->disorderRatio) {
*bind_ts = startTime + getTSRandTail(
stbInfo->timeStampStep, k,
stbInfo->disorderRatio,
@ -5811,6 +5832,7 @@ static int32_t prepareStbStmt(
if (!sourceRand) {
(*pSamplePos) ++;
}
if (recordFrom >= insertRows) {
break;
}
@ -5820,6 +5842,43 @@ static int32_t prepareStbStmt(
free(bindArray);
return k;
}
static int32_t prepareStbStmtInterlace(
SSuperTable *stbInfo,
TAOS_STMT *stmt,
char *tableName, uint32_t batch,
uint64_t insertRows,
uint64_t recordFrom,
int64_t startTime,
int64_t *pSamplePos)
{
return prepareStbStmt(
stbInfo,
stmt,
tableName,
g_args.num_of_RPR,
insertRows, 0, startTime,
pSamplePos);
}
static int32_t prepareStbStmtProgressive(
SSuperTable *stbInfo,
TAOS_STMT *stmt,
char *tableName, uint32_t batch,
uint64_t insertRows,
uint64_t recordFrom,
int64_t startTime,
int64_t *pSamplePos)
{
return prepareStbStmt(
stbInfo,
stmt,
tableName,
g_args.num_of_RPR,
insertRows, recordFrom, startTime,
pSamplePos);
}
#endif
static int32_t generateStbProgressiveData(
@ -5890,10 +5949,12 @@ static int32_t generateProgressiveDataWithoutStb(
static void printStatPerThread(threadInfo *pThreadInfo)
{
fprintf(stderr, "====thread[%d] completed total inserted rows: %"PRIu64 ", total affected rows: %"PRIu64". %.2f records/second====\n",
pThreadInfo->threadID,
pThreadInfo->totalInsertRows,
pThreadInfo->totalAffectedRows,
(pThreadInfo->totalDelay)?(double)((pThreadInfo->totalAffectedRows / (pThreadInfo->totalDelay)/1000.0)): FLT_MAX);
pThreadInfo->threadID,
pThreadInfo->totalInsertRows,
pThreadInfo->totalAffectedRows,
(pThreadInfo->totalDelay/1000.0)?
(double)(pThreadInfo->totalAffectedRows/(pThreadInfo->totalDelay/1000.0)):
FLT_MAX);
}
// sync write interlace data
@ -5992,7 +6053,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) {
uint32_t recOfBatch = 0;
for (uint32_t i = 0; i < batchPerTblTimes; i ++) {
for (uint64_t i = 0; i < batchPerTblTimes; i ++) {
char tableName[TSDB_TABLE_NAME_LEN];
getTableName(tableName, pThreadInfo, tableSeq);
@ -6009,7 +6070,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) {
if (superTblInfo) {
if (superTblInfo->iface == STMT_IFACE) {
#if STMT_IFACE_ENABLED == 1
generated = prepareStbStmt(
generated = prepareStbStmtInterlace(
superTblInfo,
pThreadInfo->stmt,
tableName,
@ -6238,7 +6299,7 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) {
if (superTblInfo) {
if (superTblInfo->iface == STMT_IFACE) {
#if STMT_IFACE_ENABLED == 1
generated = prepareStbStmt(
generated = prepareStbStmtProgressive(
superTblInfo,
pThreadInfo->stmt,
tableName,
@ -6668,13 +6729,24 @@ static void startMultiThreadInsertData(int threads, char* db_name,
char buffer[3000];
char *pstr = buffer;
pstr += sprintf(pstr, "INSERT INTO ? values(?");
if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) {
pstr += sprintf(pstr, "INSERT INTO ? USING %s TAGS(?",
superTblInfo->sTblName);
for (int tag = 0; tag < (superTblInfo->tagCount - 1); tag ++ ) {
pstr += sprintf(pstr, ",?");
}
pstr += sprintf(pstr, ") VALUES(?");
} else {
pstr += sprintf(pstr, "INSERT INTO ? VALUES(?");
}
for (int col = 0; col < columnCount; col ++) {
pstr += sprintf(pstr, ",?");
}
pstr += sprintf(pstr, ")");
debugPrint("%s() LN%d, buffer: %s", __func__, __LINE__, buffer);
int ret = taos_stmt_prepare(pThreadInfo->stmt, buffer, 0);
if (ret != 0){
errorPrint("failed to execute taos_stmt_prepare. return 0x%x. reason: %s\n",
@ -6758,34 +6830,40 @@ static void startMultiThreadInsertData(int threads, char* db_name,
int64_t end = taosGetTimestampMs();
int64_t t = end - start;
double tInMs = t/1000.0;
if (superTblInfo) {
fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n",
t / 1000.0, superTblInfo->totalInsertRows,
tInMs, superTblInfo->totalInsertRows,
superTblInfo->totalAffectedRows,
threads, db_name, superTblInfo->sTblName,
(double)superTblInfo->totalInsertRows / (t / 1000.0));
(tInMs)?
(double)(superTblInfo->totalInsertRows/tInMs):FLT_MAX);
if (g_fpOfInsertResult) {
fprintf(g_fpOfInsertResult,
"Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n",
t / 1000.0, superTblInfo->totalInsertRows,
tInMs, superTblInfo->totalInsertRows,
superTblInfo->totalAffectedRows,
threads, db_name, superTblInfo->sTblName,
(double)superTblInfo->totalInsertRows / (t / 1000.0));
(tInMs)?
(double)(superTblInfo->totalInsertRows/tInMs):FLT_MAX);
}
} else {
fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n",
t / 1000.0, g_args.totalInsertRows,
tInMs, g_args.totalInsertRows,
g_args.totalAffectedRows,
threads, db_name,
(double)g_args.totalInsertRows / (t / 1000.0));
(tInMs)?
(double)(g_args.totalInsertRows/tInMs):FLT_MAX);
if (g_fpOfInsertResult) {
fprintf(g_fpOfInsertResult,
"Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n",
t * 1000.0, g_args.totalInsertRows,
tInMs, g_args.totalInsertRows,
g_args.totalAffectedRows,
threads, db_name,
(double)g_args.totalInsertRows / (t / 1000.0));
(tInMs)?
(double)(g_args.totalInsertRows/tInMs):FLT_MAX);
}
}

19
src/mnode/inc/mnodeDef.h
View File

@ -214,6 +214,23 @@ typedef struct SUserObj {
struct SAcctObj * pAcct;
} SUserObj;
typedef struct SFuncObj {
char name[TSDB_FUNC_NAME_LEN];
char path[128];
int32_t contLen;
char cont[TSDB_FUNC_CODE_LEN];
int32_t funcType;
int32_t bufSize;
int64_t createdTime;
uint8_t resType;
int16_t resBytes;
int64_t sig; // partial md5 sign
int16_t type; // [lua script|so|js]
int8_t reserved[64];
int8_t updateEnd[4];
int32_t refCount;
} SFuncObj;
typedef struct {
int64_t totalStorage; // Total storage wrtten from this account
int64_t compStorage; // Compressed storage on disk
@ -258,7 +275,7 @@ typedef struct {
void * pIter;
void ** ppShow;
int16_t offset[TSDB_MAX_COLUMNS];
int16_t bytes[TSDB_MAX_COLUMNS];
int32_t bytes[TSDB_MAX_COLUMNS];
int32_t numOfReads;
int8_t maxReplica;
int8_t reserved0[1];

40
src/mnode/inc/mnodeFunc.h Normal file
View File

@ -0,0 +1,40 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_MNODE_FUNC_H
#define TDENGINE_MNODE_FUNC_H
#ifdef __cplusplus
extern "C" {
#endif
#include "mnodeDef.h"
int32_t mnodeInitFuncs();
void mnodeCleanupFuncs();
SFuncObj *mnodeGetFunc(char *name);
void * mnodeGetNextFunc(void *pIter, SFuncObj **pFunc);
void mnodeCancelGetNextFunc(void *pIter);
void mnodeIncFuncRef(SFuncObj *pFunc);
void mnodeDecFuncRef(SFuncObj *pFunc);
int32_t mnodeCreateFunc(SAcctObj *pAcct, char *name, int32_t codeLen, char *code, char *path, uint8_t outputType, int16_t outputLen, int32_t funcType, int32_t bufSize, SMnodeMsg *pMsg);
#ifdef __cplusplus
}
#endif
#endif

5
src/mnode/inc/mnodeSdb.h
View File

@ -33,7 +33,8 @@ typedef enum {
SDB_TABLE_VGROUP = 6,
SDB_TABLE_STABLE = 7,
SDB_TABLE_CTABLE = 8,
SDB_TABLE_MAX = 9
SDB_TABLE_FUNC = 9,
SDB_TABLE_MAX = 10
} ESdbTable;
typedef enum {
@ -112,4 +113,4 @@ int32_t mnodeCompactWal();
}
#endif
#endif
#endif

3
src/mnode/src/mnodeDb.c
View File

@ -554,6 +554,9 @@ void mnodeCleanupDbs() {
tsDbSdb = NULL;
}
static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
int32_t cols = 0;

485
src/mnode/src/mnodeFunc.c Normal file
View File

@ -0,0 +1,485 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "trpc.h"
#include "tutil.h"
#include "tglobal.h"
#include "tgrant.h"
#include "tdataformat.h"
#include "tkey.h"
#include "mnode.h"
#include "dnode.h"
#include "mnodeDef.h"
#include "mnodeInt.h"
#include "mnodeAcct.h"
#include "mnodeUser.h"
#include "mnodeMnode.h"
#include "mnodeSdb.h"
#include "mnodeShow.h"
#include "mnodeFunc.h"
#include "mnodeWrite.h"
#include "mnodeRead.h"
#include "mnodePeer.h"
int64_t tsFuncRid = -1;
static void * tsFuncSdb = NULL;
static int32_t tsFuncUpdateSize = 0;
static int32_t mnodeGetFuncMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t mnodeRetrieveFuncs(SShowObj *pShow, char *data, int32_t rows, void *pConn);
static int32_t mnodeProcessRetrieveFuncImplMsg(SMnodeMsg *pMsg);
static int32_t mnodeProcessCreateFuncMsg(SMnodeMsg *pMsg);
static int32_t mnodeProcessDropFuncMsg(SMnodeMsg *pMsg);
static int32_t mnodeFuncActionDestroy(SSdbRow *pRow) {
tfree(pRow->pObj);
return TSDB_CODE_SUCCESS;
}
static int32_t mnodeFuncActionInsert(SSdbRow *pRow) {
SFuncObj *pFunc = pRow->pObj;
mTrace("func:%s, contLen: %d, insert into sdb", pFunc->name, pFunc->contLen);
return TSDB_CODE_SUCCESS;
}
static int32_t mnodeFuncActionDelete(SSdbRow *pRow) {
SFuncObj *pFunc = pRow->pObj;
mTrace("func:%s, length: %d, delete from sdb", pFunc->name, pFunc->contLen);
return TSDB_CODE_SUCCESS;
}
static int32_t mnodeFuncActionUpdate(SSdbRow *pRow) {
SFuncObj *pFunc = pRow->pObj;
SFuncObj *pSaved = mnodeGetFunc(pFunc->name);
if (pFunc != pSaved) {
memcpy(pSaved, pFunc, tsFuncUpdateSize);
free(pFunc);
}
mnodeDecFuncRef(pSaved);
return TSDB_CODE_SUCCESS;
}
static int32_t mnodeFuncActionEncode(SSdbRow *pRow) {
SFuncObj *pFunc = pRow->pObj;
memcpy(pRow->rowData, pFunc, tsFuncUpdateSize);
pRow->rowSize = tsFuncUpdateSize;
return TSDB_CODE_SUCCESS;
}
static int32_t mnodeFuncActionDecode(SSdbRow *pRow) {
SFuncObj *pFunc = (SFuncObj *)calloc(1, sizeof(SFuncObj));
if (pFunc == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY;
memcpy(pFunc, pRow->rowData, tsFuncUpdateSize);
pRow->pObj = pFunc;
return TSDB_CODE_SUCCESS;
}
static int32_t mnodeFuncActionRestored() {
int64_t numOfRows = sdbGetNumOfRows(tsFuncSdb);
if (numOfRows <= 0 && dnodeIsFirstDeploy()) {
mInfo("dnode first deploy, func restored.");
}
return TSDB_CODE_SUCCESS;
}
int32_t mnodeInitFuncs() {
SFuncObj tObj;
tsFuncUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj);
SSdbTableDesc desc = {
.id = SDB_TABLE_FUNC,
.name = "funcs",
.hashSessions = TSDB_DEFAULT_USERS_HASH_SIZE,
.maxRowSize = tsFuncUpdateSize,
.refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj),
.keyType = SDB_KEY_STRING,
.fpInsert = mnodeFuncActionInsert,
.fpDelete = mnodeFuncActionDelete,
.fpUpdate = mnodeFuncActionUpdate,
.fpEncode = mnodeFuncActionEncode,
.fpDecode = mnodeFuncActionDecode,
.fpDestroy = mnodeFuncActionDestroy,
.fpRestored = mnodeFuncActionRestored
};
tsFuncRid = sdbOpenTable(&desc);
tsFuncSdb = sdbGetTableByRid(tsFuncRid);
if (tsFuncSdb == NULL) {
mError("table:%s, failed to create hash", desc.name);
return -1;
}
mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_CREATE_FUNCTION, mnodeProcessCreateFuncMsg);
mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_DROP_FUNCTION, mnodeProcessDropFuncMsg);
mnodeAddReadMsgHandle(TSDB_MSG_TYPE_CM_RETRIEVE_FUNC, mnodeProcessRetrieveFuncImplMsg);
mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_FUNCTION, mnodeGetFuncMeta);
mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_FUNCTION, mnodeRetrieveFuncs);
mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_FUNCTION, mnodeCancelGetNextFunc);
mDebug("table:%s, hash is created", desc.name);
return 0;
}
void mnodeCleanupFuncs() {
sdbCloseTable(tsFuncRid);
tsFuncSdb = NULL;
}
SFuncObj *mnodeGetFunc(char *name) {
return (SFuncObj *)sdbGetRow(tsFuncSdb, name);
}
void *mnodeGetNextFunc(void *pIter, SFuncObj **pFunc) {
return sdbFetchRow(tsFuncSdb, pIter, (void **)pFunc);
}
void mnodeCancelGetNextFunc(void *pIter) {
sdbFreeIter(tsFuncSdb, pIter);
}
void mnodeIncFuncRef(SFuncObj *pFunc) {
sdbIncRef(tsFuncSdb, pFunc);
}
void mnodeDecFuncRef(SFuncObj *pFunc) {
sdbDecRef(tsFuncSdb, pFunc);
}
/*
static int32_t mnodeUpdateFunc(SFuncObj *pFunc, void *pMsg) {
SSdbRow row = {
.type = SDB_OPER_GLOBAL,
.pTable = tsFuncSdb,
.pObj = pFunc,
.pMsg = pMsg
};
int32_t code = sdbUpdateRow(&row);
if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
mError("func:%s, failed to alter by %s, reason:%s", pFunc->name, mnodeGetUserFromMsg(pMsg), tstrerror(code));
} else {
mLInfo("func:%s, is altered by %s", pFunc->name, mnodeGetUserFromMsg(pMsg));
}
return code;
}
*/
int32_t mnodeCreateFunc(SAcctObj *pAcct, char *name, int32_t codeLen, char *codeScript, char *path, uint8_t outputType, int16_t outputLen, int32_t funcType, int32_t bufSize, SMnodeMsg *pMsg) {
if (grantCheck(TSDB_GRANT_TIME) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_GRANT_EXPIRED;
}
if (!pMsg->pUser->writeAuth) {
return TSDB_CODE_MND_NO_RIGHTS;
}
int32_t code = acctCheck(pAcct, ACCT_GRANT_USER);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
code = grantCheck(TSDB_GRANT_USER);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
if (name[0] == 0) {
return TSDB_CODE_MND_INVALID_FUNC_NAME;
}
if (codeScript[0] == 0) {
return TSDB_CODE_MND_INVALID_FUNC_CODE;
}
if (codeLen < 0 || codeLen > TSDB_FUNC_CODE_LEN) {
return TSDB_CODE_MND_INVALID_FUNC_CODE;
}
if (bufSize < 0 || bufSize > TSDB_FUNC_BUF_SIZE) {
return TSDB_CODE_MND_INVALID_FUNC_BUFSIZE;
}
SFuncObj *pFunc = mnodeGetFunc(name);
if (pFunc != NULL) {
mDebug("func:%s, is already there", name);
mnodeDecFuncRef(pFunc);
return TSDB_CODE_MND_FUNC_ALREADY_EXIST;
}
pFunc = calloc(1, sizeof(SFuncObj));
tstrncpy(pFunc->name, name, TSDB_FUNC_NAME_LEN);
tstrncpy(pFunc->path, path, tListLen(pFunc->path));
memcpy(pFunc->cont, codeScript, codeLen);
pFunc->contLen = codeLen;
pFunc->createdTime = taosGetTimestampMs();
pFunc->resType = outputType;
pFunc->resBytes = outputLen;
pFunc->funcType = funcType;
pFunc->bufSize = bufSize;
pFunc->sig = 0;
pFunc->type = 1; //lua script, refactor
SSdbRow row = {
.type = SDB_OPER_GLOBAL,
.pTable = tsFuncSdb,
.pObj = pFunc,
.rowSize = sizeof(SFuncObj),
.pMsg = pMsg
};
code = sdbInsertRow(&row);
if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
mError("func:%s, failed to create by %s, reason:%s", pFunc->name, mnodeGetUserFromMsg(pMsg), tstrerror(code));
tfree(pFunc);
} else {
mLInfo("func:%s, is created by %s", pFunc->name, mnodeGetUserFromMsg(pMsg));
}
return code;
}
static int32_t mnodeDropFunc(SFuncObj *pFunc, void *pMsg) {
SSdbRow row = {
.type = SDB_OPER_GLOBAL,
.pTable = tsFuncSdb,
.pObj = pFunc,
.pMsg = pMsg
};
int32_t code = sdbDeleteRow(&row);
if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
mError("func:%s, failed to drop by %s, reason:%s", pFunc->name, mnodeGetUserFromMsg(pMsg), tstrerror(code));
} else {
mLInfo("func:%s, is dropped by %s", pFunc->name, mnodeGetUserFromMsg(pMsg));
}
return code;
}
static int32_t mnodeGetFuncsNum() {
return (int32_t)sdbGetNumOfRows(tsFuncSdb);
}
static int32_t mnodeGetFuncMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
SUserObj *pUser = mnodeGetUserFromConn(pConn);
if (pUser == NULL) {
return TSDB_CODE_MND_NO_USER_FROM_CONN;
}
int32_t cols = 0;
SSchema *pSchema = pMeta->schema;
pShow->bytes[cols] = TSDB_FUNC_NAME_LEN + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "name");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = PATH_MAX + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "path");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "aggregate");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = TSDB_TYPE_STR_MAX_LEN + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "outputtype");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 8;
pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
strcpy(pSchema[cols].name, "create_time");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "code_len");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "bufsize");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pMeta->numOfColumns = htons(cols);
strcpy(pMeta->tableFname, "show funcs");
pShow->numOfColumns = cols;
pShow->offset[0] = 0;
for (int32_t i = 1; i < cols; ++i) {
pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
}
pShow->numOfRows = mnodeGetFuncsNum();
pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
mnodeDecUserRef(pUser);
return 0;
}
static void* mnodeGenTypeStr(char *buf, int32_t buflen, uint8_t type, int16_t len) {
char *msg = "unknown";
if (type >= sizeof(tDataTypes)/sizeof(tDataTypes[0])) {
return msg;
}
if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) {
int32_t bytes = len > 0 ? (int)(len - VARSTR_HEADER_SIZE) : len;
snprintf(buf, buflen - 1, "%s(%d)", tDataTypes[type].name, type == TSDB_DATA_TYPE_NCHAR ? bytes/4 : bytes);
buf[buflen - 1] = 0;
return buf;
}
return tDataTypes[type].name;
}
static int32_t mnodeRetrieveFuncs(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
int32_t numOfRows = 0;
SFuncObj *pFunc = NULL;
int32_t cols = 0;
char *pWrite;
char buf[TSDB_TYPE_STR_MAX_LEN];
while (numOfRows < rows) {
pShow->pIter = mnodeGetNextFunc(pShow->pIter, &pFunc);
if (pFunc == NULL) break;
cols = 0;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pFunc->name, pShow->bytes[cols]);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pFunc->path, pShow->bytes[cols]);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int32_t *)pWrite = pFunc->funcType == TSDB_UDF_TYPE_AGGREGATE ? 1 : 0;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, mnodeGenTypeStr(buf, TSDB_TYPE_STR_MAX_LEN, pFunc->resType, pFunc->resBytes), pShow->bytes[cols]);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int64_t *)pWrite = pFunc->createdTime;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int32_t *)pWrite = pFunc->contLen;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int32_t *)pWrite = pFunc->bufSize;
cols++;
numOfRows++;
mnodeDecFuncRef(pFunc);
}
mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
pShow->numOfReads += numOfRows;
return numOfRows;
}
static int32_t mnodeProcessCreateFuncMsg(SMnodeMsg *pMsg) {
SCreateFuncMsg *pCreate = pMsg->rpcMsg.pCont;
pCreate->codeLen = htonl(pCreate->codeLen);
pCreate->outputLen = htons(pCreate->outputLen);
pCreate->funcType = htonl(pCreate->funcType);
pCreate->bufSize = htonl(pCreate->bufSize);
return mnodeCreateFunc(pMsg->pUser->pAcct, pCreate->name, pCreate->codeLen, pCreate->code, pCreate->path, pCreate->outputType, pCreate->outputLen, pCreate->funcType, pCreate->bufSize, pMsg);
}
static int32_t mnodeProcessDropFuncMsg(SMnodeMsg *pMsg) {
SDropFuncMsg *pDrop = pMsg->rpcMsg.pCont;
SFuncObj *pFunc = mnodeGetFunc(pDrop->name);
if (pFunc == NULL) {
return TSDB_CODE_MND_INVALID_FUNC;
}
return mnodeDropFunc(pFunc, pMsg);
}
static int32_t mnodeProcessRetrieveFuncImplMsg(SMnodeMsg *pMsg) {
SRetrieveFuncMsg *pInfo = pMsg->rpcMsg.pCont;
pInfo->num = htonl(pInfo->num);
int32_t t = sizeof(SUdfFuncMsg) + (sizeof(SFunctionInfoMsg) + TSDB_FUNC_CODE_LEN) * pInfo->num + 16384;
SUdfFuncMsg *pFuncMsg = rpcMallocCont(t);
pFuncMsg->num = htonl(pInfo->num);
char* pOutput = pFuncMsg->content;
tstr* name = (tstr*) pInfo->name;
for(int32_t i = 0; i < pInfo->num; ++i) {
char buf[TSDB_FUNC_NAME_LEN] = {0};
tstrncpy(buf, name->data, htons(name->len) + 1);
SFuncObj* pFuncObj = mnodeGetFunc(buf);
if (pFuncObj == NULL) {
mError("function %s does not exist", buf);
return TSDB_CODE_MND_INVALID_FUNC;
}
SFunctionInfoMsg* pFuncInfo = (SFunctionInfoMsg*) pOutput;
strcpy(pFuncInfo->name, buf);
pFuncInfo->len = htonl(pFuncObj->contLen);
memcpy(pFuncInfo->content, pFuncObj->cont, pFuncObj->contLen);
pFuncInfo->funcType = htonl(pFuncObj->funcType);
pFuncInfo->resType = pFuncObj->resType;
pFuncInfo->resBytes = htons(pFuncObj->resBytes);
pFuncInfo->bufSize = htonl(pFuncObj->bufSize);
pOutput += sizeof(SFunctionInfoMsg) + pFuncObj->contLen;
name =(tstr *)((char *)name + sizeof(*name) + htons(name->len));
}
pMsg->rpcRsp.rsp = pFuncMsg;
pMsg->rpcRsp.len = (int32_t)(pOutput - (char*)pFuncMsg);
return TSDB_CODE_SUCCESS;
}

2
src/mnode/src/mnodeMain.c
View File

@ -32,6 +32,7 @@
#include "mnodeSdb.h"
#include "mnodeVgroup.h"
#include "mnodeUser.h"
#include "mnodeFunc.h"
#include "mnodeTable.h"
#include "mnodeCluster.h"
#include "mnodeShow.h"
@ -46,6 +47,7 @@ static SStep tsMnodeSteps[] = {
{"cluster", mnodeInitCluster, mnodeCleanupCluster},
{"accts", mnodeInitAccts, mnodeCleanupAccts},
{"users", mnodeInitUsers, mnodeCleanupUsers},
{"funcs", mnodeInitFuncs, mnodeCleanupFuncs},
{"dnodes", mnodeInitDnodes, mnodeCleanupDnodes},
{"dbs", mnodeInitDbs, mnodeCleanupDbs},
{"vgroups", mnodeInitVgroups, mnodeCleanupVgroups},

47
src/mnode/src/mnodeTable.c
View File

@ -42,6 +42,7 @@
#include "mnodeWrite.h"
#include "mnodeRead.h"
#include "mnodePeer.h"
#include "mnodeFunc.h"
#define ALTER_CTABLE_RETRY_TIMES 3
#define CREATE_CTABLE_RETRY_TIMES 10
@ -2926,6 +2927,7 @@ static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) {
pInfo->numOfTables = htonl(pInfo->numOfTables);
pInfo->numOfVgroups = htonl(pInfo->numOfVgroups);
pInfo->numOfUdfs = htonl(pInfo->numOfUdfs);
int32_t contLen = pMsg->rpcMsg.contLen - sizeof(SMultiTableInfoMsg);
@ -2934,9 +2936,9 @@ static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) {
char* str = strndup(pInfo->tableNames, contLen);
char** nameList = strsplit(str, ",", &num);
SArray* pList = taosArrayInit(4, POINTER_BYTES);
SMultiTableMeta *pMultiMeta = NULL;
if (num != pInfo->numOfTables + pInfo->numOfVgroups) {
SMultiTableMeta *pMultiMeta = NULL;
if (num != pInfo->numOfTables + pInfo->numOfVgroups + pInfo->numOfUdfs) {
mError("msg:%p, app:%p, failed to get multi-tableMeta, msg inconsistent", pMsg, pMsg->rpcMsg.ahandle);
code = TSDB_CODE_MND_INVALID_TABLE_NAME;
goto _end;
@ -3011,8 +3013,10 @@ static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) {
}
}
int32_t tableNum = pInfo->numOfTables + pInfo->numOfVgroups;
// add the additional super table names that needs the vgroup info
for(;t < num; ++t) {
for(;t < tableNum; ++t) {
taosArrayPush(pList, &nameList[t]);
}
@ -3043,6 +3047,36 @@ static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) {
pMultiMeta->contLen = (int32_t) (msg - (char*) pMultiMeta);
pMultiMeta->numOfTables = htonl(pMultiMeta->numOfTables);
// add the user-defined-function information
for(int32_t i = 0; i < pInfo->numOfUdfs; ++i, ++t) {
char buf[TSDB_FUNC_NAME_LEN] = {0};
strcpy(buf, nameList[t]);
SFuncObj* pFuncObj = mnodeGetFunc(buf);
if (pFuncObj == NULL) {
mError("function %s does not exist", buf);
code = TSDB_CODE_MND_INVALID_FUNC;
goto _end;
}
SFunctionInfoMsg* pFuncInfo = (SFunctionInfoMsg*) msg;
strcpy(pFuncInfo->name, buf);
pFuncInfo->len = htonl(pFuncObj->contLen);
memcpy(pFuncInfo->content, pFuncObj->cont, pFuncObj->contLen);
pFuncInfo->funcType = htonl(pFuncObj->funcType);
pFuncInfo->resType = pFuncObj->resType;
pFuncInfo->resBytes = htons(pFuncObj->resBytes);
pFuncInfo->bufSize = htonl(pFuncObj->bufSize);
msg += sizeof(SFunctionInfoMsg) + pFuncObj->contLen;
}
pMultiMeta->contLen = (int32_t) (msg - (char*) pMultiMeta);
pMultiMeta->numOfUdf = htonl(pInfo->numOfUdfs);
pMsg->rpcRsp.rsp = pMultiMeta;
pMsg->rpcRsp.len = pMultiMeta->contLen;
code = TSDB_CODE_SUCCESS;
@ -3053,11 +3087,12 @@ static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) {
goto _end;
}
int32_t len = tsCompressString(pMultiMeta->meta, (int32_t)pMultiMeta->contLen - sizeof(SMultiTableMeta), 1,
tmp + sizeof(SMultiTableMeta), (int32_t)pMultiMeta->contLen - sizeof(SMultiTableMeta) + 2, ONE_STAGE_COMP, NULL, 0);
int32_t dataLen = (int32_t)pMultiMeta->contLen - sizeof(SMultiTableMeta);
int32_t len = tsCompressString(pMultiMeta->meta, dataLen, 1, tmp + sizeof(SMultiTableMeta), (int32_t)dataLen + 2,
ONE_STAGE_COMP, NULL, 0);
pMultiMeta->rawLen = pMultiMeta->contLen;
if (len == -1 || len + sizeof(SMultiTableMeta) >= pMultiMeta->contLen + 2) { // compress failed, do not compress this binary data
if (len == -1 || len >= dataLen + 2) { // compress failed, do not compress this binary data
pMultiMeta->compressed = 0;
memcpy(tmp, pMultiMeta, sizeof(SMultiTableMeta) + pMultiMeta->contLen);
} else {

1
src/os/inc/os.h
View File

@ -37,6 +37,7 @@ extern "C" {
#include "osSysinfo.h"
#include "osTime.h"
#include "osTimer.h"
#include "osSystem.h"
void osInit();

0
src/os/inc/osArm32.h Normal file
View File

0
src/os/inc/osArm64.h Normal file
View File

3
src/os/inc/osInc.h
View File

@ -76,7 +76,8 @@ extern "C" {
#include <dispatch/dispatch.h>
#include "osEok.h"
#else
#include <argp.h>
#include <argp.h>
#include <dlfcn.h>
#include <endian.h>
#include <linux/sysctl.h>
#include <poll.h>

0
src/os/inc/osLinux32.h Normal file
View File

0
src/os/inc/osLinux64.h Normal file
View File

31
src/os/inc/osSystem.h Normal file
View File

@ -0,0 +1,31 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_OS_SYSTEM_H
#define TDENGINE_OS_SYSTEM_H
#ifdef __cplusplus
extern "C" {
#endif
void* taosLoadDll(const char *filename);
void* taosLoadSym(void* handle, char* name);
void taosCloseDll(void *handle);
#ifdef __cplusplus
}
#endif
#endif

32
src/os/src/darwin/darwinSystem.c Normal file
View File

@ -0,0 +1,32 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "tglobal.h"
#include "tulog.h"
void* taosLoadDll(const char *filename) {
return NULL;
}
void* taosLoadSym(void* handle, char* name) {
return NULL;
}
void taosCloseDll(void *handle) {
}

2
src/os/src/linux/CMakeLists.txt
View File

@ -4,4 +4,4 @@ PROJECT(TDengine)
AUX_SOURCE_DIRECTORY(. SRC)
ADD_LIBRARY(oslinux ${SRC})
TARGET_LINK_LIBRARIES(oslinux m rt z)
TARGET_LINK_LIBRARIES(oslinux m rt z dl)

54
src/os/src/linux/osSystem.c Normal file
View File

@ -0,0 +1,54 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "tconfig.h"
#include "tglobal.h"
#include "tulog.h"
void* taosLoadDll(const char *filename) {
void *handle = dlopen (filename, RTLD_LAZY);
if (!handle) {
uError("load dll:%s failed, error:%s", filename, dlerror());
return NULL;
}
uDebug("dll %s loaded", filename);
return handle;
}
void* taosLoadSym(void* handle, char* name) {
void* sym = dlsym(handle, name);
char* error = NULL;
if ((error = dlerror()) != NULL) {
uWarn("load sym:%s failed, error:%s", name, dlerror());
return NULL;
}
uDebug("sym %s loaded", name)
return sym;
}
void taosCloseDll(void *handle) {
if (handle) {
dlclose(handle);
}
}

32
src/os/src/windows/wSystem.c Normal file
View File

@ -0,0 +1,32 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "tglobal.h"
#include "tulog.h"
void* taosLoadDll(const char *filename) {
return NULL;
}
void* taosLoadSym(void* handle, char* name) {
return NULL;
}
void taosCloseDll(void *handle) {
}

2
src/plugins/http/inc/httpParser.h
View File

@ -17,7 +17,7 @@
#define HTTP_PARSER_H
#include "httpGzip.h"
#define HTTP_MAX_URL 5 // http url stack size
#define HTTP_MAX_URL 6 // http url stack size
#define HTTP_CODE_CONTINUE 100
#define HTTP_CODE_SWITCHING_PROTOCOL 101

86
src/plugins/http/src/httpRestHandle.c
View File

@ -119,6 +119,90 @@ bool restProcessSqlRequest(HttpContext* pContext, int32_t timestampFmt) {
return true;
}
#define REST_FUNC_URL_POS 2
#define REST_OUTP_URL_POS 3
#define REST_AGGR_URL_POS 4
#define REST_BUFF_URL_POS 5
#define HTTP_FUNC_LEN 32
#define HTTP_OUTP_LEN 16
#define HTTP_AGGR_LEN 2
#define HTTP_BUFF_LEN 32
static int udfSaveFile(const char *fname, const char *content, long len) {
int fd = open(fname, O_WRONLY | O_CREAT | O_EXCL | O_BINARY, 0755);
if (fd < 0)
return -1;
if (taosWrite(fd, (void *)content, len) < 0)
return -1;
return 0;
}
static bool restProcessUdfRequest(HttpContext* pContext) {
HttpParser* pParser = pContext->parser;
if (pParser->path[REST_FUNC_URL_POS].pos >= HTTP_FUNC_LEN || pParser->path[REST_FUNC_URL_POS].pos <= 0) {
return false;
}
if (pParser->path[REST_OUTP_URL_POS].pos >= HTTP_OUTP_LEN || pParser->path[REST_OUTP_URL_POS].pos <= 0) {
return false;
}
if (pParser->path[REST_AGGR_URL_POS].pos >= HTTP_AGGR_LEN || pParser->path[REST_AGGR_URL_POS].pos <= 0) {
return false;
}
if (pParser->path[REST_BUFF_URL_POS].pos >= HTTP_BUFF_LEN || pParser->path[REST_BUFF_URL_POS].pos <= 0) {
return false;
}
char* sql = pContext->parser->body.str;
int len = pContext->parser->body.pos;
if (sql == NULL) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_SQL_INPUT);
return false;
}
char udfDir[256] = {0};
char buf[64] = "udf-";
char funcName[64] = {0};
int aggr = 0;
char outputType[16] = {0};
char buffSize[32] = {0};
tstrncpy(funcName, pParser->path[REST_FUNC_URL_POS].str, HTTP_FUNC_LEN);
tstrncpy(buf + 4, funcName, HTTP_FUNC_LEN);
if (pParser->path[REST_AGGR_URL_POS].str[0] != '0') {
aggr = 1;
}
tstrncpy(outputType, pParser->path[REST_OUTP_URL_POS].str, HTTP_OUTP_LEN);
tstrncpy(buffSize, pParser->path[REST_BUFF_URL_POS].str, HTTP_BUFF_LEN);
taosGetTmpfilePath(funcName, udfDir);
udfSaveFile(udfDir, sql, len);
tfree(sql);
pContext->parser->body.str = malloc(1024);
sql = pContext->parser->body.str;
int sqlLen = sprintf(sql, "create %s function %s as \"%s\" outputtype %s bufsize %s",
aggr == 1 ? "aggregate" : " ", funcName, udfDir, outputType, buffSize);
pContext->parser->body.pos = sqlLen;
pContext->parser->body.size = sqlLen + 1;
HttpSqlCmd* cmd = &(pContext->singleCmd);
cmd->nativSql = sql;
pContext->reqType = HTTP_REQTYPE_SINGLE_SQL;
pContext->encodeMethod = &restEncodeSqlLocalTimeStringMethod;
return true;
}
bool restProcessRequest(struct HttpContext* pContext) {
if (httpUrlMatch(pContext, REST_ACTION_URL_POS, "login")) {
restGetUserFromUrl(pContext);
@ -138,6 +222,8 @@ bool restProcessRequest(struct HttpContext* pContext) {
return restProcessSqlRequest(pContext, REST_TIMESTAMP_FMT_UTC_STRING);
} else if (httpUrlMatch(pContext, REST_ACTION_URL_POS, "login")) {
return restProcessLoginRequest(pContext);
} else if (httpUrlMatch(pContext, REST_ACTION_URL_POS, "udf")) {
return restProcessUdfRequest(pContext);
} else {
}

6
src/query/CMakeLists.txt
View File

@ -8,14 +8,14 @@ INCLUDE_DIRECTORIES(inc)
AUX_SOURCE_DIRECTORY(src SRC)
ADD_LIBRARY(query ${SRC})
SET_SOURCE_FILES_PROPERTIES(src/sql.c PROPERTIES COMPILE_FLAGS -w)
TARGET_LINK_LIBRARIES(query tsdb tutil)
TARGET_LINK_LIBRARIES(query tsdb tutil lua)
IF (TD_LINUX)
TARGET_LINK_LIBRARIES(query m rt)
TARGET_LINK_LIBRARIES(query m rt lua)
ADD_SUBDIRECTORY(tests)
ENDIF ()
IF (TD_DARWIN)
TARGET_LINK_LIBRARIES(query m)
TARGET_LINK_LIBRARIES(query m lua)
ADD_SUBDIRECTORY(tests)
ENDIF ()

11
src/query/inc/qAggMain.h
View File

@ -27,6 +27,7 @@ extern "C" {
#include "trpc.h"
#include "tvariant.h"
#include "tsdb.h"
#include "qUdf.h"
#define TSDB_FUNC_INVALID_ID -1
#define TSDB_FUNC_COUNT 0
@ -201,10 +202,8 @@ typedef struct SAggFunctionInfo {
int8_t stableFuncId; // transfer function for super table query
uint16_t status;
bool (*init)(SQLFunctionCtx *pCtx); // setup the execute environment
bool (*init)(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultCellInfo); // setup the execute environment
void (*xFunction)(SQLFunctionCtx *pCtx); // blocks version function
// void (*xFunctionF)(SQLFunctionCtx *pCtx, int32_t position); // single-row function version, todo merge with blockwise function
// finalizer must be called after all xFunction has been executed to generated final result.
void (*xFinalize)(SQLFunctionCtx *pCtx);
@ -216,7 +215,7 @@ typedef struct SAggFunctionInfo {
#define GET_RES_INFO(ctx) ((ctx)->resultInfo)
int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, int16_t *type,
int16_t *len, int32_t *interBytes, int16_t extLength, bool isSuperTable);
int16_t *len, int32_t *interBytes, int16_t extLength, bool isSuperTable, SUdfInfo* pUdfInfo);
int32_t isValidFunction(const char* name, int32_t len);
#define IS_STREAM_QUERY_VALID(x) (((x)&TSDB_FUNCSTATE_STREAM) != 0)
@ -262,9 +261,9 @@ bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const cha
static FORCE_INLINE void initResultInfo(SResultRowCellInfo *pResInfo, int32_t bufLen) {
pResInfo->initialized = true; // the this struct has been initialized flag
pResInfo->complete = false;
pResInfo->complete = false;
pResInfo->hasResult = false;
pResInfo->numOfRes = 0;
pResInfo->numOfRes = 0;
memset(GET_ROWCELL_INTERBUF(pResInfo), 0, bufLen);
}

18
src/query/inc/qExecutor.h
View File

@ -29,6 +29,7 @@
#include "tarray.h"
#include "tlockfree.h"
#include "tsdb.h"
#include "qUdf.h"
struct SColumnFilterElem;
typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, const char* val1, const char* val2, int16_t type);
@ -257,6 +258,7 @@ typedef struct SQueryAttr {
SMemRef memRef;
STableGroupInfo tableGroupInfo; // table <tid, last_key> list SArray<STableKeyInfo>
int32_t vgId;
SArray *pUdfInfo; // no need to free
} SQueryAttr;
typedef SSDataBlock* (*__operator_fn_t)(void* param, bool* newgroup);
@ -297,6 +299,7 @@ typedef struct SQueryRuntimeEnv {
STableQueryInfo *current;
SRspResultInfo resultInfo;
SHashObj *pTableRetrieveTsMap;
SUdfInfo *pUdfInfo;
} SQueryRuntimeEnv;
enum {
@ -392,6 +395,7 @@ typedef struct SQueryParam {
SGroupbyExpr *pGroupbyExpr;
int32_t tableScanOperator;
SArray *pOperator;
SUdfInfo *pUdfInfo;
} SQueryParam;
typedef struct STableScanInfo {
@ -531,6 +535,7 @@ typedef struct SMultiwayMergeInfo {
bool hasPrev;
bool groupMix;
SArray *udfInfo;
} SMultiwayMergeInfo;
void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream);
@ -553,8 +558,8 @@ SOperatorInfo* createDistinctOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperat
SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv);
SOperatorInfo* createMultiwaySortOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput,
int32_t numOfRows, void* merger, bool groupMix);
SOperatorInfo* createGlobalAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* param, SArray* pUdfInfo);
SOperatorInfo* createStatewindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createGlobalAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* param);
SOperatorInfo* createSLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* merger);
SOperatorInfo* createFilterOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr,
int32_t numOfOutput, SColumnInfo* pCols, int32_t numOfFilter);
@ -578,18 +583,19 @@ void setInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlo
int32_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput);
void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset);
void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOfInputRows);
void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity);
void freeParam(SQueryParam *param);
int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param);
int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExprInfo** pExprInfo,
SSqlExpr** pExprMsg, SColumnInfo* pTagCols, int32_t queryType, void* pMsg);
SSqlExpr** pExprMsg, SColumnInfo* pTagCols, int32_t queryType, void* pMsg, SUdfInfo* pUdfInfo);
int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t numOfOutput, SExprInfo **pExprInfo,
SSqlExpr **pExpr, SExprInfo *prevExpr);
SSqlExpr **pExpr, SExprInfo *prevExpr, SUdfInfo *pUdfInfo);
SGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pColIndex, int32_t *code);
SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs,
SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, int32_t vgId, char* sql, uint64_t qId);
SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, int32_t vgId, char* sql, uint64_t qId, SUdfInfo* pUdfInfo);
int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* pQInfo, SQueryParam* param, char* start,
int32_t prevResultLen, void* merger);
@ -608,6 +614,8 @@ bool doBuildResCheck(SQInfo* pQInfo);
void setQueryStatus(SQueryRuntimeEnv *pRuntimeEnv, int8_t status);
bool onlyQueryTags(SQueryAttr* pQueryAttr);
void destroyUdfInfo(SUdfInfo* pUdfInfo);
bool isValidQInfo(void *param);
int32_t doDumpQueryResult(SQInfo *pQInfo, char *data);
@ -625,4 +633,6 @@ void freeQueryAttr(SQueryAttr *pQuery);
int32_t getMaximumIdleDurationSec();
void doInvokeUdf(SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, int32_t type);
#endif // TDENGINE_QEXECUTOR_H

82
src/query/inc/qScript.h Normal file
View File

@ -0,0 +1,82 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_QSCRIPT_H
#define TDENGINE_QSCRIPT_H
#include <lua.h>
#include <lauxlib.h>
#include <lualib.h>
#include "tutil.h"
#include "hash.h"
#include "tlist.h"
#include "qUdf.h"
#define MAX_FUNC_NAME 64
#define USER_FUNC_NAME "funcName"
#define USER_FUNC_NAME_LIMIT 48
enum ScriptState {
SCRIPT_STATE_INIT,
SCRIPT_STATE_ADD,
SCRIPT_STATE_MERGE,
SCRIPT_STATE_FINALIZE
};
typedef struct {
SHashObj *funcId; //func already registed in lua_env, may be no use
lua_State *lua_state; // lua env
} ScriptEnv;
typedef struct ScriptCtx {
char funcName[USER_FUNC_NAME_LIMIT];
int8_t state;
ScriptEnv *pEnv;
int8_t isAgg; // agg function or not
// init value of udf script
int8_t resType;
int16_t resBytes;
int32_t numOfOutput;
int32_t offset;
} ScriptCtx;
int taosLoadScriptInit(void *pInit);
void taosLoadScriptNormal(void *pInit, char *pInput, int16_t iType, int16_t iBytes, int32_t numOfRows,
int64_t *ptsList, int64_t key, char* pOutput, char *ptsOutput, int32_t *numOfOutput, int16_t oType, int16_t oBytes);
void taosLoadScriptFinalize(void *pInit, int64_t key, char *pOutput, int32_t *output);
void taosLoadScriptMerge(void *pCtx, char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput);
void taosLoadScriptDestroy(void *pInit);
typedef struct {
SList *scriptEnvs; //
int32_t mSize; // pool limit
int32_t cSize; // current available size
pthread_mutex_t mutex;
} ScriptEnvPool;
ScriptCtx* createScriptCtx(char *str, int8_t resType, int16_t resBytes);
void destroyScriptCtx(void *pScriptCtx);
int32_t scriptEnvPoolInit();
void scriptEnvPoolCleanup();
bool isValidScript(char *script, int32_t len);
#endif //TDENGINE_QSCRIPT_H

20
src/query/inc/qSqlparser.h
View File

@ -182,6 +182,15 @@ typedef struct SCreateDbInfo {
int16_t partitions;
} SCreateDbInfo;
typedef struct SCreateFuncInfo {
SStrToken name;
SStrToken path;
int32_t type;
int32_t bufSize;
TAOS_FIELD output;
} SCreateFuncInfo;
typedef struct SCreateAcctInfo {
int32_t maxUsers;
int32_t maxDbs;
@ -214,10 +223,11 @@ typedef struct SMiscInfo {
int16_t tableType;
SUserInfo user;
union {
SCreateDbInfo dbOpt;
SCreateAcctInfo acctOpt;
SShowInfo showOpt;
SStrToken id;
SCreateDbInfo dbOpt;
SCreateAcctInfo acctOpt;
SCreateFuncInfo funcOpt;
SShowInfo showOpt;
SStrToken id;
};
} SMiscInfo;
@ -226,6 +236,7 @@ typedef struct SSqlInfo {
bool valid;
SArray *list; // todo refactor
char msg[256];
SArray *funcs;
union {
SCreateTableSql *pCreateTableInfo;
SAlterTableInfo *pAlterInfo;
@ -271,6 +282,7 @@ SRelationInfo *addSubqueryElem(SRelationInfo* pRelationInfo, SArray* pSub, SStrT
// sql expr leaf node
tSqlExpr *tSqlExprCreateIdValue(SStrToken *pToken, int32_t optrType);
tSqlExpr *tSqlExprCreateFunction(SArray *pParam, SStrToken *pFuncToken, SStrToken *endToken, int32_t optType);
SArray *tStrTokenAppend(SArray *pList, SStrToken *pToken);
tSqlExpr *tSqlExprCreate(tSqlExpr *pLeft, tSqlExpr *pRight, int32_t optrType);
tSqlExpr *tSqlExprClone(tSqlExpr *pSrc);

9
src/query/inc/qTableMeta.h
View File

@ -125,8 +125,12 @@ typedef struct SQueryInfo {
int32_t round; // 0/1/....
int32_t bufLen;
char* buf;
struct SQInfo* pQInfo; // global merge operator
struct SQueryAttr* pQueryAttr; // query object
bool udfCopy;
SArray *pUdfInfo;
struct SQInfo *pQInfo; // global merge operator
struct SQueryAttr *pQueryAttr; // query object
struct SQueryInfo *sibling; // sibling
SArray *pUpstream; // SArray<struct SQueryInfo>
@ -141,6 +145,7 @@ typedef struct SQueryInfo {
bool onlyTagQuery;
bool orderProjectQuery;
bool stateWindow;
bool globalMerge;
} SQueryInfo;
/**

74
src/query/inc/qUdf.h Normal file
View File

@ -0,0 +1,74 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_QUDF_H
#define TDENGINE_QUDF_H
enum { TSDB_UDF_FUNC_NORMAL = 0, TSDB_UDF_FUNC_INIT, TSDB_UDF_FUNC_FINALIZE, TSDB_UDF_FUNC_MERGE, TSDB_UDF_FUNC_DESTROY, TSDB_UDF_FUNC_MAX_NUM };
typedef struct SUdfInit{
int32_t maybe_null; /* 1 if function can return NULL */
uint32_t decimals; /* for real functions */
uint64_t length; /* For string functions */
char *ptr; /* free pointer for function data */
int32_t const_item; /* 0 if result is independent of arguments */
// script like lua/javascript
void* script_ctx;
void (*destroyCtxFunc)(void *script_ctx);
} SUdfInit;
typedef struct SUdfInfo {
int32_t functionId; // system assigned function id
int32_t funcType; // scalar function or aggregate function
int8_t resType; // result type
int16_t resBytes; // result byte
int32_t contLen; // content length
int32_t bufSize; //interbuf size
char *name; // function name
void *handle; // handle loaded in mem
void *funcs[TSDB_UDF_FUNC_MAX_NUM]; // function ptr
// for script like lua/javascript only
int isScript;
void *pScriptCtx;
SUdfInit init;
char *content;
char *path;
} SUdfInfo;
//script
typedef int32_t (*scriptInitFunc)(void *pCtx);
typedef void (*scriptNormalFunc)(void *pCtx, char* data, int16_t iType, int16_t iBytes, int32_t numOfRows,
int64_t* ptList, int64_t key, char* dataOutput, char* tsOutput, int32_t* numOfOutput, int16_t oType, int16_t oBytes);
typedef void (*scriptFinalizeFunc)(void *pCtx, int64_t key, char* dataOutput, int32_t* numOfOutput);
typedef void (*scriptMergeFunc)(void *pCtx, char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput);
typedef void (*scriptDestroyFunc)(void* pCtx);
// dynamic lib
typedef void (*udfNormalFunc)(char* data, int16_t itype, int16_t iBytes, int32_t numOfRows, int64_t* ts, char* dataOutput, char* interBuf,
char* tsOutput, int32_t* numOfOutput, int16_t oType, int16_t oBytes, SUdfInit* buf);
typedef int32_t (*udfInitFunc)(SUdfInit* data);
typedef void (*udfFinalizeFunc)(char* dataOutput, char *interBuf, int32_t* numOfOutput, SUdfInit* buf);
typedef void (*udfMergeFunc)(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf);
typedef void (*udfDestroyFunc)(SUdfInit* buf);
#endif // TDENGINE_QUDF_H

2
src/query/inc/qUtil.h
View File

@ -105,4 +105,6 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo);
int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, SQueryRuntimeEnv *pRuntimeEnv, int32_t* offset);
int32_t initUdfInfo(SUdfInfo* pUdfInfo);
#endif // TDENGINE_QUERYUTIL_H

11
src/query/inc/sql.y
View File

@ -65,6 +65,7 @@ program ::= cmd. {}
//////////////////////////////////THE SHOW STATEMENT///////////////////////////////////////////
cmd ::= SHOW DATABASES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_DB, 0, 0);}
cmd ::= SHOW TOPICS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_TP, 0, 0);}
cmd ::= SHOW FUNCTIONS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_FUNCTION, 0, 0);}
cmd ::= SHOW MNODES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_MNODE, 0, 0);}
cmd ::= SHOW DNODES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_DNODE, 0, 0);}
cmd ::= SHOW ACCOUNTS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_ACCT, 0, 0);}
@ -147,6 +148,7 @@ cmd ::= DROP STABLE ifexists(Y) ids(X) cpxName(Z). {
cmd ::= DROP DATABASE ifexists(Y) ids(X). { setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &X, &Y, TSDB_DB_TYPE_DEFAULT, -1); }
cmd ::= DROP TOPIC ifexists(Y) ids(X). { setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &X, &Y, TSDB_DB_TYPE_TOPIC, -1); }
cmd ::= DROP FUNCTION ids(X). { setDropFuncInfo(pInfo, TSDB_SQL_DROP_FUNCTION, &X); }
cmd ::= DROP DNODE ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_DROP_DNODE, 1, &X); }
cmd ::= DROP USER ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_DROP_USER, 1, &X); }
@ -200,8 +202,13 @@ cmd ::= CREATE ACCOUNT ids(X) PASS ids(Y) acct_optr(Z).
{ setCreateAcctSql(pInfo, TSDB_SQL_CREATE_ACCT, &X, &Y, &Z);}
cmd ::= CREATE DATABASE ifnotexists(Z) ids(X) db_optr(Y). { setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &X, &Y, &Z);}
cmd ::= CREATE TOPIC ifnotexists(Z) ids(X) topic_optr(Y). { setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &X, &Y, &Z);}
cmd ::= CREATE FUNCTION ids(X) AS ids(Y) OUTPUTTYPE typename(Z) bufsize(B). { setCreateFuncInfo(pInfo, TSDB_SQL_CREATE_FUNCTION, &X, &Y, &Z, &B, 1);}
cmd ::= CREATE AGGREGATE FUNCTION ids(X) AS ids(Y) OUTPUTTYPE typename(Z) bufsize(B). { setCreateFuncInfo(pInfo, TSDB_SQL_CREATE_FUNCTION, &X, &Y, &Z, &B, 2);}
cmd ::= CREATE USER ids(X) PASS ids(Y). { setCreateUserSql(pInfo, &X, &Y);}
bufsize(Y) ::= . { Y.n = 0; }
bufsize(Y) ::= BUFSIZE INTEGER(X). { Y = X; }
pps(Y) ::= . { Y.n = 0; }
pps(Y) ::= PPS INTEGER(X). { Y = X; }
@ -704,10 +711,10 @@ expr(A) ::= BOOL(X). { A = tSqlExprCreateIdValue(&X, TK_BOOL);}
expr(A) ::= NULL(X). { A = tSqlExprCreateIdValue(&X, TK_NULL);}
// ordinary functions: min(x), max(x), top(k, 20)
expr(A) ::= ID(X) LP exprlist(Y) RP(E). { A = tSqlExprCreateFunction(Y, &X, &E, X.type); }
expr(A) ::= ID(X) LP exprlist(Y) RP(E). { tStrTokenAppend(pInfo->funcs, &X); A = tSqlExprCreateFunction(Y, &X, &E, X.type); }
// for parsing sql functions with wildcard for parameters. e.g., count(*)/first(*)/last(*) operation
expr(A) ::= ID(X) LP STAR RP(Y). { A = tSqlExprCreateFunction(NULL, &X, &Y, X.type); }
expr(A) ::= ID(X) LP STAR RP(Y). { tStrTokenAppend(pInfo->funcs, &X); A = tSqlExprCreateFunction(NULL, &X, &Y, X.type); }
// is (not) null expression
expr(A) ::= expr(X) IS NULL. {A = tSqlExprCreate(X, NULL, TK_ISNULL);}

122
src/query/src/qAggMain.c
View File

@ -27,6 +27,7 @@
#include "qPercentile.h"
#include "qTsbuf.h"
#include "queryLog.h"
#include "qUdf.h"
#define GET_INPUT_DATA_LIST(x) ((char *)((x)->pInput))
#define GET_INPUT_DATA(x, y) (GET_INPUT_DATA_LIST(x) + (y) * (x)->inputBytes)
@ -169,12 +170,13 @@ typedef struct SDerivInfo {
} SDerivInfo;
int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, int16_t *type,
int16_t *bytes, int32_t *interBytes, int16_t extLength, bool isSuperTable) {
int16_t *bytes, int32_t *interBytes, int16_t extLength, bool isSuperTable, SUdfInfo* pUdfInfo) {
if (!isValidDataType(dataType)) {
qError("Illegal data type %d or data type length %d", dataType, dataBytes);
return TSDB_CODE_TSC_INVALID_OPERATION;
}
if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || functionId == TSDB_FUNC_TAG_DUMMY ||
functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_TAGPRJ ||
functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_INTERP) {
@ -234,6 +236,20 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
}
if (isSuperTable) {
if (functionId < 0) {
if (pUdfInfo->bufSize > 0) {
*type = TSDB_DATA_TYPE_BINARY;
*bytes = pUdfInfo->bufSize;
*interBytes = *bytes;
} else {
*type = pUdfInfo->resType;
*bytes = pUdfInfo->resBytes;
*interBytes = *bytes;
}
return TSDB_CODE_SUCCESS;
}
if (functionId == TSDB_FUNC_MIN || functionId == TSDB_FUNC_MAX) {
*type = TSDB_DATA_TYPE_BINARY;
*bytes = (int16_t)(dataBytes + DATA_SET_FLAG_SIZE);
@ -288,7 +304,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
return TSDB_CODE_SUCCESS;
}
}
if (functionId == TSDB_FUNC_SUM) {
if (IS_SIGNED_NUMERIC_TYPE(dataType)) {
*type = TSDB_DATA_TYPE_BIGINT;
@ -313,7 +329,20 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
*interBytes = sizeof(STwaInfo);
return TSDB_CODE_SUCCESS;
}
if (functionId < 0) {
*type = pUdfInfo->resType;
*bytes = pUdfInfo->resBytes;
if (pUdfInfo->bufSize > 0) {
*interBytes = pUdfInfo->bufSize;
} else {
*interBytes = *bytes;
}
return TSDB_CODE_SUCCESS;
}
if (functionId == TSDB_FUNC_AVG) {
*type = TSDB_DATA_TYPE_DOUBLE;
*bytes = sizeof(double);
@ -390,14 +419,13 @@ int32_t isValidFunction(const char* name, int32_t len) {
return -1;
}
static bool function_setup(SQLFunctionCtx *pCtx) {
SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
if (pResInfo->initialized) {
static bool function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) {
if (pResultInfo->initialized) {
return false;
}
memset(pCtx->pOutput, 0, (size_t)pCtx->outputBytes);
initResultInfo(pResInfo, pCtx->interBufBytes);
initResultInfo(pResultInfo, pCtx->interBufBytes);
return true;
}
@ -985,8 +1013,8 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin,
}
}
static bool min_func_setup(SQLFunctionCtx *pCtx) {
if (!function_setup(pCtx)) {
static bool min_func_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) {
if (!function_setup(pCtx, pResultInfo)) {
return false; // not initialized since it has been initialized
}
@ -1030,8 +1058,8 @@ static bool min_func_setup(SQLFunctionCtx *pCtx) {
return true;
}
static bool max_func_setup(SQLFunctionCtx *pCtx) {
if (!function_setup(pCtx)) {
static bool max_func_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) {
if (!function_setup(pCtx, pResultInfo)) {
return false; // not initialized since it has been initialized
}
@ -1435,8 +1463,8 @@ static void stddev_dst_finalizer(SQLFunctionCtx *pCtx) {
}
//////////////////////////////////////////////////////////////////////////////////////
static bool first_last_function_setup(SQLFunctionCtx *pCtx) {
if (!function_setup(pCtx)) {
static bool first_last_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) {
if (!function_setup(pCtx, pResInfo)) {
return false;
}
@ -1606,7 +1634,7 @@ static void last_function(SQLFunctionCtx *pCtx) {
break;
}
}
SET_VAL(pCtx, notNullElems, 1);
}
@ -2007,7 +2035,7 @@ static void buildTopBotStruct(STopBotInfo *pTopBotInfo, SQLFunctionCtx *pCtx) {
char *tmp = (char *)pTopBotInfo + sizeof(STopBotInfo);
pTopBotInfo->res = (tValuePair**) tmp;
tmp += POINTER_BYTES * pCtx->param[0].i64;
size_t size = sizeof(tValuePair) + pCtx->tagInfo.tagsLen;
for (int32_t i = 0; i < pCtx->param[0].i64; ++i) {
@ -2073,14 +2101,13 @@ bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const cha
}
}
static bool top_bottom_function_setup(SQLFunctionCtx *pCtx) {
if (!function_setup(pCtx)) {
static bool top_bottom_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) {
if (!function_setup(pCtx, pResInfo)) {
return false;
}
STopBotInfo *pInfo = getTopBotOutputInfo(pCtx);
buildTopBotStruct(pInfo, pCtx);
return true;
}
@ -2229,14 +2256,13 @@ static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) {
}
///////////////////////////////////////////////////////////////////////////////////////////////
static bool percentile_function_setup(SQLFunctionCtx *pCtx) {
if (!function_setup(pCtx)) {
static bool percentile_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) {
if (!function_setup(pCtx, pResultInfo)) {
return false;
}
// in the first round, get the min-max value of all involved data
SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResultInfo);
SET_DOUBLE_VAL(&pInfo->minval, DBL_MAX);
SET_DOUBLE_VAL(&pInfo->maxval, -DBL_MAX);
pInfo->numOfElems = 0;
@ -2367,8 +2393,8 @@ static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) {
return pInfo;
}
static bool apercentile_function_setup(SQLFunctionCtx *pCtx) {
if (!function_setup(pCtx)) {
static bool apercentile_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) {
if (!function_setup(pCtx, pResultInfo)) {
return false;
}
@ -2477,12 +2503,11 @@ static void apercentile_finalizer(SQLFunctionCtx *pCtx) {
}
/////////////////////////////////////////////////////////////////////////////////
static bool leastsquares_function_setup(SQLFunctionCtx *pCtx) {
if (!function_setup(pCtx)) {
static bool leastsquares_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) {
if (!function_setup(pCtx, pResInfo)) {
return false;
}
SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx);
SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
// 2*3 matrix
@ -2706,8 +2731,8 @@ enum {
INITIAL_VALUE_NOT_ASSIGNED = 0,
};
static bool diff_function_setup(SQLFunctionCtx *pCtx) {
if (!function_setup(pCtx)) {
static bool diff_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) {
if (!function_setup(pCtx, pResInfo)) {
return false;
}
@ -2716,14 +2741,13 @@ static bool diff_function_setup(SQLFunctionCtx *pCtx) {
return false;
}
static bool deriv_function_setup(SQLFunctionCtx *pCtx) {
if (!function_setup(pCtx)) {
static bool deriv_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) {
if (!function_setup(pCtx, pResultInfo)) {
return false;
}
// diff function require the value is set to -1
SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
SDerivInfo* pDerivInfo = GET_ROWCELL_INTERBUF(pResInfo);
SDerivInfo* pDerivInfo = GET_ROWCELL_INTERBUF(pResultInfo);
pDerivInfo->ignoreNegative = pCtx->param[1].i64;
pDerivInfo->prevTs = -1;
@ -3130,12 +3154,12 @@ static void arithmetic_function(SQLFunctionCtx *pCtx) {
}
/////////////////////////////////////////////////////////////////////////////////
static bool spread_function_setup(SQLFunctionCtx *pCtx) {
if (!function_setup(pCtx)) {
static bool spread_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) {
if (!function_setup(pCtx, pResInfo)) {
return false;
}
SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
// this is the server-side setup function in client-side, the secondary merge do not need this procedure
if (pCtx->currentStage == MERGE_STAGE) {
@ -3289,12 +3313,10 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) {
* param[2]: end time
* @param pCtx
*/
static bool twa_function_setup(SQLFunctionCtx *pCtx) {
if (!function_setup(pCtx)) {
static bool twa_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) {
if (!function_setup(pCtx, pResInfo)) {
return false;
}
SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
pInfo->p.key = INT64_MIN;
@ -3726,14 +3748,12 @@ static void interp_function(SQLFunctionCtx *pCtx) {
}
}
static bool ts_comp_function_setup(SQLFunctionCtx *pCtx) {
if (!function_setup(pCtx)) {
static bool ts_comp_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) {
if (!function_setup(pCtx, pResInfo)) {
return false; // not initialized since it has been initialized
}
SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
STSCompInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
pInfo->pTSBuf = tsBufCreate(false, pCtx->order);
pInfo->pTSBuf->tsOrder = pCtx->order;
return true;
@ -3816,13 +3836,11 @@ static double do_calc_rate(const SRateInfo* pRateInfo, double tickPerSec) {
return (duration > 0)? ((double)diff) / (duration/tickPerSec):0.0;
}
static bool rate_function_setup(SQLFunctionCtx *pCtx) {
if (!function_setup(pCtx)) {
static bool rate_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) {
if (!function_setup(pCtx, pResInfo)) {
return false;
}
SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
SRateInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
pInfo->correctionValue = 0;
pInfo->firstKey = INT64_MIN;
@ -3920,7 +3938,7 @@ static void irate_function(SQLFunctionCtx *pCtx) {
int32_t notNullElems = 0;
SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo);
TSKEY *primaryKey = GET_TS_LIST(pCtx);
for (int32_t i = pCtx->size - 1; i >= 0; --i) {
char *pData = GET_INPUT_DATA(pCtx, i);
if (pCtx->hasNull && isNull(pData, pCtx->inputType)) {

431
src/query/src/qExecutor.c
View File

@ -29,10 +29,11 @@
#include "ttype.h"
#include "tcompare.h"
#include "tscompression.h"
#include "qScript.h"
#define IS_MASTER_SCAN(runtime) ((runtime)->scanFlag == MASTER_SCAN)
#define IS_REVERSE_SCAN(runtime) ((runtime)->scanFlag == REVERSE_SCAN)
#define IS_REPEAT_SCAN(runtime) ((runtime)->scanFlag == REPEAT_SCAN)
#define IS_REPEAT_SCAN(runtime) ((runtime)->scanFlag == REPEAT_SCAN)
#define SET_MASTER_SCAN_FLAG(runtime) ((runtime)->scanFlag = MASTER_SCAN)
#define SET_REVERSE_SCAN_FLAG(runtime) ((runtime)->scanFlag = REVERSE_SCAN)
@ -212,24 +213,24 @@ SArray* getOrderCheckColumns(SQueryAttr* pQuery);
typedef struct SRowCompSupporter {
SQueryRuntimeEnv *pRuntimeEnv;
SQueryRuntimeEnv *pRuntimeEnv;
int16_t dataOffset;
__compar_fn_t comFunc;
__compar_fn_t comFunc;
} SRowCompSupporter;
static int compareRowData(const void *a, const void *b, const void *userData) {
const SResultRow *pRow1 = (const SResultRow *)a;
const SResultRow *pRow1 = (const SResultRow *)a;
const SResultRow *pRow2 = (const SResultRow *)b;
SRowCompSupporter *supporter = (SRowCompSupporter *)userData;
SQueryRuntimeEnv* pRuntimeEnv = supporter->pRuntimeEnv;
SRowCompSupporter *supporter = (SRowCompSupporter *)userData;
SQueryRuntimeEnv* pRuntimeEnv = supporter->pRuntimeEnv;
tFilePage *page1 = getResBufPage(pRuntimeEnv->pResultBuf, pRow1->pageId);
tFilePage *page2 = getResBufPage(pRuntimeEnv->pResultBuf, pRow2->pageId);
int16_t offset = supporter->dataOffset;
int16_t offset = supporter->dataOffset;
char *in1 = getPosInResultPage(pRuntimeEnv->pQueryAttr, page1, pRow1->offset, offset);
char *in2 = getPosInResultPage(pRuntimeEnv->pQueryAttr, page2, pRow2->offset, offset);
char *in2 = getPosInResultPage(pRuntimeEnv->pQueryAttr, page2, pRow2->offset, offset);
return (in1 != NULL && in2 != NULL) ? supporter->comFunc(in1, in2) : 0;
}
@ -250,7 +251,7 @@ static void sortGroupResByOrderList(SGroupResInfo *pGroupResInfo, SQueryRuntimeE
bool found = false;
int16_t dataOffset = 0;
for (int32_t j = 0; j < pDataBlock->info.numOfCols; ++j) {
SColumnInfoData* pColInfoData = (SColumnInfoData *)taosArrayGet(pDataBlock->pDataBlock, j);
if (orderId == j) {
@ -259,15 +260,15 @@ static void sortGroupResByOrderList(SGroupResInfo *pGroupResInfo, SQueryRuntimeE
}
dataOffset += pColInfoData->info.bytes;
}
}
if (found == false) {
return;
}
int16_t type = pRuntimeEnv->pQueryAttr->pExpr1[orderId].base.resType;
SRowCompSupporter support = {.pRuntimeEnv = pRuntimeEnv, .dataOffset = dataOffset, .comFunc = getComparFunc(type, 0)};
SRowCompSupporter support = {.pRuntimeEnv = pRuntimeEnv, .dataOffset = dataOffset, .comFunc = getComparFunc(type, 0)};
taosArraySortPWithExt(pGroupResInfo->pRows, compareRowData, &support);
}
@ -780,6 +781,79 @@ static int32_t getNumOfRowsInTimeWindow(SQueryRuntimeEnv* pRuntimeEnv, SDataBloc
return num;
}
void doInvokeUdf(SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, int32_t type) {
int32_t output = 0;
if (pUdfInfo == NULL || pUdfInfo->funcs[type] == NULL) {
qError("empty udf function, type:%d", type);
return;
}
qDebug("invoke udf function:%s,%p", pUdfInfo->name, pUdfInfo->funcs[type]);
switch (type) {
case TSDB_UDF_FUNC_NORMAL:
if (pUdfInfo->isScript) {
(*(scriptNormalFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL])(pUdfInfo->pScriptCtx,
(char *)pCtx->pInput + idx * pCtx->inputType, pCtx->inputType, pCtx->inputBytes, pCtx->size, pCtx->ptsList, pCtx->startTs, pCtx->pOutput,
(char *)pCtx->ptsOutputBuf, &output, pCtx->outputType, pCtx->outputBytes);
} else {
SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
void *interBuf = (void *)GET_ROWCELL_INTERBUF(pResInfo);
(*(udfNormalFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL])((char *)pCtx->pInput + idx * pCtx->inputType, pCtx->inputType, pCtx->inputBytes, pCtx->size, pCtx->ptsList,
pCtx->pOutput, interBuf, (char *)pCtx->ptsOutputBuf, &output, pCtx->outputType, pCtx->outputBytes, &pUdfInfo->init);
}
if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) {
pCtx->resultInfo->numOfRes = output;
} else {
pCtx->resultInfo->numOfRes += output;
}
if (pCtx->resultInfo->numOfRes > 0) {
pCtx->resultInfo->hasResult = DATA_SET_FLAG;
}
break;
case TSDB_UDF_FUNC_MERGE:
if (pUdfInfo->isScript) {
(*(scriptMergeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE])(pUdfInfo->pScriptCtx, pCtx->pInput, pCtx->size, pCtx->pOutput, &output);
} else {
(*(udfMergeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE])(pCtx->pInput, pCtx->size, pCtx->pOutput, &output, &pUdfInfo->init);
}
// set the output value exist
pCtx->resultInfo->numOfRes = output;
if (output > 0) {
pCtx->resultInfo->hasResult = DATA_SET_FLAG;
}
break;
case TSDB_UDF_FUNC_FINALIZE: {
SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
void *interBuf = (void *)GET_ROWCELL_INTERBUF(pResInfo);
if (pUdfInfo->isScript) {
(*(scriptFinalizeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE])(pUdfInfo->pScriptCtx, pCtx->startTs, pCtx->pOutput, &output);
} else {
(*(udfFinalizeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE])(pCtx->pOutput, interBuf, &output, &pUdfInfo->init);
}
// set the output value exist
pCtx->resultInfo->numOfRes = output;
if (output > 0) {
pCtx->resultInfo->hasResult = DATA_SET_FLAG;
}
break;
}
}
return;
}
static void doApplyFunctions(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, STimeWindow* pWin, int32_t offset,
int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput) {
SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr;
@ -789,7 +863,7 @@ static void doApplyFunctions(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx
pCtx[k].size = forwardStep;
pCtx[k].startTs = pWin->skey;
// keep it temprarily
// keep it temporarialy
char* start = pCtx[k].pInput;
int32_t pos = (QUERY_IS_ASC_QUERY(pQueryAttr)) ? offset : offset - (forwardStep - 1);
@ -807,8 +881,14 @@ static void doApplyFunctions(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx
pCtx[k].preAggVals.isSet = false;
}
int32_t functionId = pCtx[k].functionId;
if (functionNeedToExecute(pRuntimeEnv, &pCtx[k])) {
aAggs[pCtx[k].functionId].xFunction(&pCtx[k]);
if (functionId < 0) { // load the script and exec, pRuntimeEnv->pUdfInfo
SUdfInfo* pUdfInfo = pRuntimeEnv->pUdfInfo;
doInvokeUdf(pUdfInfo, &pCtx[k], 0, TSDB_UDF_FUNC_NORMAL);
} else {
aAggs[functionId].xFunction(&pCtx[k]);
}
}
// restore it
@ -1018,6 +1098,13 @@ static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx,
pCtx[i].pInput = p->pData;
assert(p->info.colId == pColIndex->colId && pCtx[i].inputType == p->info.type);
if (pCtx[i].functionId < 0) {
SColumnInfoData* tsInfo = taosArrayGet(pBlock->pDataBlock, 0);
pCtx[i].ptsList = (int64_t*) tsInfo->pData;
continue;
}
uint32_t status = aAggs[pCtx[i].functionId].status;
if ((status & (TSDB_FUNCSTATE_SELECTIVITY | TSDB_FUNCSTATE_NEED_TS)) != 0) {
SColumnInfoData* tsInfo = taosArrayGet(pBlock->pDataBlock, 0);
@ -1050,7 +1137,14 @@ static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SQLFunction
for (int32_t k = 0; k < pOperator->numOfOutput; ++k) {
if (functionNeedToExecute(pRuntimeEnv, &pCtx[k])) {
pCtx[k].startTs = startTs;// this can be set during create the struct
aAggs[pCtx[k].functionId].xFunction(&pCtx[k]);
int32_t functionId = pCtx[k].functionId;
if (functionId < 0) {
SUdfInfo* pUdfInfo = pRuntimeEnv->pUdfInfo;
doInvokeUdf(pUdfInfo, &pCtx[k], 0, TSDB_UDF_FUNC_NORMAL);
} else {
aAggs[functionId].xFunction(&pCtx[k]);
}
}
}
}
@ -1066,7 +1160,15 @@ static void projectApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx
pCtx[k].order = TSDB_ORDER_ASC;
}
aAggs[pCtx[k].functionId].xFunction(&pCtx[k]);
pCtx[k].startTs = pQueryAttr->window.skey;
if (pCtx[k].functionId < 0) {
// load the script and exec
SUdfInfo* pUdfInfo = pRuntimeEnv->pUdfInfo;
doInvokeUdf(pUdfInfo, &pCtx[k], 0, TSDB_UDF_FUNC_NORMAL);
} else {
aAggs[pCtx[k].functionId].xFunction(&pCtx[k]);
}
}
}
@ -1953,7 +2055,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOf
case OP_GlobalAggregate: {
pRuntimeEnv->proot = createGlobalAggregateOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr3,
pQueryAttr->numOfExpr3, merger);
pQueryAttr->numOfExpr3, merger, pQueryAttr->pUdfInfo);
break;
}
@ -2024,6 +2126,8 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) {
tfree(pRuntimeEnv->sasArray);
}
destroyUdfInfo(pRuntimeEnv->pUdfInfo);
destroyResultBuf(pRuntimeEnv->pResultBuf);
doFreeQueryHandle(pRuntimeEnv);
@ -2234,7 +2338,7 @@ static bool onlyFirstQuery(SQueryAttr *pQueryAttr) { return onlyOneQueryType(pQu
static bool onlyLastQuery(SQueryAttr *pQueryAttr) { return onlyOneQueryType(pQueryAttr, TSDB_FUNC_LAST, TSDB_FUNC_LAST_DST); }
static bool notContainSessionOrStateWindow(SQueryAttr *pQueryAttr) { return !(pQueryAttr->sw.gap > 0 || pQueryAttr->stateWindow); }
static bool notContainSessionOrStateWindow(SQueryAttr *pQueryAttr) { return !(pQueryAttr->sw.gap > 0 || pQueryAttr->stateWindow); }
static int32_t updateBlockLoadStatus(SQueryAttr *pQuery, int32_t status) {
bool hasFirstLastFunc = false;
@ -2444,14 +2548,14 @@ static bool doFilterByBlockStatistics(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis
}
SDataStatis* pDataBlockst = &pDataStatis[index];
if (pFilterInfo->info.type == TSDB_DATA_TYPE_FLOAT) {
float minval = (float)(*(double *)(&pDataBlockst->min));
float maxval = (float)(*(double *)(&pDataBlockst->max));
for (int32_t i = 0; i < pFilterInfo->numOfFilters; ++i) {
if (pFilterInfo->pFilters[i].filterInfo.lowerRelOptr == TSDB_RELATION_IN) {
continue;
continue;
}
ret &= pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&minval, (char *)&maxval, TSDB_DATA_TYPE_FLOAT);
if (ret == false) {
@ -2461,9 +2565,9 @@ static bool doFilterByBlockStatistics(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis
} else {
for (int32_t i = 0; i < pFilterInfo->numOfFilters; ++i) {
if (pFilterInfo->pFilters[i].filterInfo.lowerRelOptr == TSDB_RELATION_IN) {
continue;
continue;
}
ret &= pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&pDataBlockst->min, (char *)&pDataBlockst->max, pFilterInfo->info.type);
ret &= pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&pDataBlockst->min, (char *)&pDataBlockst->max, pFilterInfo->info.type);
if (ret == false) {
return false;
}
@ -2703,9 +2807,14 @@ static uint32_t doFilterByBlockTimeWindow(STableScanInfo* pTableScanInfo, SSData
int32_t colId = pTableScanInfo->pExpr[i].base.colInfo.colId;
// group by + first/last should not apply the first/last block filter
status |= aAggs[functionId].dataReqFunc(&pTableScanInfo->pCtx[i], &pBlock->info.window, colId);
if ((status & BLK_DATA_ALL_NEEDED) == BLK_DATA_ALL_NEEDED) {
if (functionId < 0) {
status |= BLK_DATA_ALL_NEEDED;
return status;
} else {
status |= aAggs[functionId].dataReqFunc(&pTableScanInfo->pCtx[i], &pBlock->info.window, colId);
if ((status & BLK_DATA_ALL_NEEDED) == BLK_DATA_ALL_NEEDED) {
return status;
}
}
}
@ -3010,8 +3119,8 @@ void setTagValue(SOperatorInfo* pOperatorInfo, void *pTable, SQLFunctionCtx* pCt
doSetTagValueInParam(pTable, pLocalExprInfo->base.colInfo.colId, &pCtx[idx].tag, pLocalExprInfo->base.resType,
pLocalExprInfo->base.resBytes);
if (IS_NUMERIC_TYPE(pLocalExprInfo->base.resType)
|| pLocalExprInfo->base.resType == TSDB_DATA_TYPE_BOOL
if (IS_NUMERIC_TYPE(pLocalExprInfo->base.resType)
|| pLocalExprInfo->base.resType == TSDB_DATA_TYPE_BOOL
|| pLocalExprInfo->base.resType == TSDB_DATA_TYPE_TIMESTAMP) {
memcpy(pRuntimeEnv->tagVal + offset, &pCtx[idx].tag.i64, pLocalExprInfo->base.resBytes);
} else {
@ -3299,6 +3408,21 @@ void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOf
}
}
void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity) {
SSDataBlock* pDataBlock = pBInfo->pRes;
for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) {
SColumnInfoData *pColInfo = taosArrayGet(pDataBlock->pDataBlock, i);
int32_t functionId = pBInfo->pCtx[i].functionId;
if (functionId < 0) {
memset(pBInfo->pCtx[i].pOutput, 0, pColInfo->info.bytes * (*bufCapacity));
}
}
}
void initCtxOutputBuffer(SQLFunctionCtx* pCtx, int32_t size) {
for (int32_t j = 0; j < size; ++j) {
SResultRowCellInfo* pResInfo = GET_RES_INFO(&pCtx[j]);
@ -3306,7 +3430,11 @@ void initCtxOutputBuffer(SQLFunctionCtx* pCtx, int32_t size) {
continue;
}
aAggs[pCtx[j].functionId].init(&pCtx[j]);
if (pCtx[j].functionId < 0) { // todo udf initialization
continue;
} else {
aAggs[pCtx[j].functionId].init(&pCtx[j], pCtx[j].resultInfo);
}
}
}
@ -3361,9 +3489,15 @@ void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResult
setResultOutputBuf(pRuntimeEnv, buf, pCtx, numOfOutput, rowCellInfoOffset);
for (int32_t j = 0; j < numOfOutput; ++j) {
aAggs[pCtx[j].functionId].xFinalize(&pCtx[j]);
pCtx[j].startTs = buf->win.skey;
if (pCtx[j].functionId < 0) {
doInvokeUdf(pRuntimeEnv->pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE);
} else {
aAggs[pCtx[j].functionId].xFinalize(&pCtx[j]);
}
}
/*
* set the number of output results for group by normal columns, the number of output rows usually is 1 except
* the top and bottom query
@ -3373,7 +3507,11 @@ void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResult
} else {
for (int32_t j = 0; j < numOfOutput; ++j) {
aAggs[pCtx[j].functionId].xFinalize(&pCtx[j]);
if (pCtx[j].functionId < 0) {
doInvokeUdf(pRuntimeEnv->pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE);
} else {
aAggs[pCtx[j].functionId].xFinalize(&pCtx[j]);
}
}
}
}
@ -3459,12 +3597,16 @@ void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pRe
offset += pCtx[i].outputBytes;
int32_t functionId = pCtx[i].functionId;
if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE) {
if (functionId < 0) {
continue;
}
if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) {
pCtx[i].ptsOutputBuf = pCtx[0].pOutput;
}
if (!pResInfo->initialized) {
aAggs[functionId].init(&pCtx[i]);
aAggs[functionId].init(&pCtx[i], pResInfo);
}
}
}
@ -3951,7 +4093,7 @@ void calculateOperatorProfResults(SQInfo* pQInfo) {
qDebug("QInfo:0x%"PRIx64" operator prof results hash is null", pQInfo->qId);
return;
}
SArray* opStack = taosArrayInit(32, sizeof(SOperatorStackItem));
if (opStack == NULL) {
return;
@ -4803,7 +4945,7 @@ void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInf
pTableScanInfo->pResultRowInfo = &pInfo->binfo.resultRowInfo;
pTableScanInfo->rowCellInfoOffset = pInfo->binfo.rowCellInfoOffset;
} else if (pDownstream->operatorType == OP_StateWindow) {
SStateWindowOperatorInfo* pInfo = pDownstream->info;
SStateWindowOperatorInfo* pInfo = pDownstream->info;
pTableScanInfo->pCtx = pInfo->binfo.pCtx;
pTableScanInfo->pResultRowInfo = &pInfo->binfo.resultRowInfo;
@ -4923,7 +5065,7 @@ static void destroySlimitOperatorInfo(void* param, int32_t numOfOutput) {
}
SOperatorInfo* createGlobalAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream,
SExprInfo* pExpr, int32_t numOfOutput, void* param) {
SExprInfo* pExpr, int32_t numOfOutput, void* param, SArray* pUdfInfo) {
SMultiwayMergeInfo* pInfo = calloc(1, sizeof(SMultiwayMergeInfo));
pInfo->resultRowFactor =
@ -4933,6 +5075,7 @@ SOperatorInfo* createGlobalAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv,
pInfo->pMerge = param;
pInfo->bufCapacity = 4096;
pInfo->udfInfo = pUdfInfo;
pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pInfo->bufCapacity * pInfo->resultRowFactor);
pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset);
@ -5462,7 +5605,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI
TSKEY* tsList = (TSKEY*)pTsColInfoData->pData;
if (IS_REPEAT_SCAN(pRuntimeEnv) && !pInfo->reptScan) {
pInfo->reptScan = true;
tfree(pInfo->prevData);
tfree(pInfo->prevData);
}
pInfo->numOfRows = 0;
@ -5472,17 +5615,17 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI
continue;
}
if (pInfo->prevData == NULL) {
pInfo->prevData = malloc(bytes);
pInfo->prevData = malloc(bytes);
memcpy(pInfo->prevData, val, bytes);
pInfo->numOfRows = 1;
pInfo->curWindow.skey = tsList[j];
pInfo->curWindow.ekey = tsList[j];
pInfo->start = j;
pInfo->start = j;
} else if (memcmp(pInfo->prevData, val, bytes) == 0) {
pInfo->curWindow.ekey = tsList[j];
pInfo->numOfRows += 1;
//pInfo->start = j;
//pInfo->start = j;
if (j == 0 && pInfo->start != 0) {
pInfo->numOfRows = 1;
pInfo->start = 0;
@ -5504,7 +5647,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI
memcpy(pInfo->prevData, val, bytes);
pInfo->numOfRows = 1;
pInfo->start = j;
}
}
@ -5557,10 +5700,10 @@ static SSDataBlock* doStateWindowAgg(void *param, bool* newgroup) {
setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, pQueryAttr->order.order);
if (pWindowInfo->colIndex == -1) {
pWindowInfo->colIndex = getGroupbyColumnIndex(pRuntimeEnv->pQueryAttr->pGroupbyExpr, pBlock);
}
}
doStateWindowAggImpl(pOperator, pWindowInfo, pBlock);
}
// restore the value
pQueryAttr->order.order = order;
pQueryAttr->window = win;
@ -5578,7 +5721,7 @@ static SSDataBlock* doStateWindowAgg(void *param, bool* newgroup) {
}
return pBInfo->pRes->info.rows == 0? NULL:pBInfo->pRes;
}
}
static SSDataBlock* doSessionWindowAgg(void* param, bool* newgroup) {
SOperatorInfo* pOperator = (SOperatorInfo*) param;
if (pOperator->status == OP_EXEC_DONE) {
@ -5693,7 +5836,7 @@ static SSDataBlock* hashGroupbyAggregate(void* param, bool* newgroup) {
sortGroupResByOrderList(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes);
}
toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes);
if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) {
pOperator->status = OP_EXEC_DONE;
}
@ -6088,8 +6231,8 @@ SOperatorInfo* createStatewindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOpe
pOperator->cleanup = destroyStateWindowOperatorInfo;
appendUpstream(pOperator, upstream);
return pOperator;
return pOperator;
}
SOperatorInfo* createSWindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) {
SSWindowOperatorInfo* pInfo = calloc(1, sizeof(SSWindowOperatorInfo));
@ -6099,7 +6242,7 @@ SOperatorInfo* createSWindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperato
initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT);
pInfo->prevTs = INT64_MIN;
pInfo->reptScan = false;
pInfo->reptScan = false;
SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));
pOperator->name = "SessionWindowAggOperator";
@ -6690,6 +6833,9 @@ int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param) {
pQueryMsg->sw.primaryColId = htonl(pQueryMsg->sw.primaryColId);
pQueryMsg->tableScanOperator = htonl(pQueryMsg->tableScanOperator);
pQueryMsg->numOfOperator = htonl(pQueryMsg->numOfOperator);
pQueryMsg->udfContentOffset = htonl(pQueryMsg->udfContentOffset);
pQueryMsg->udfContentLen = htonl(pQueryMsg->udfContentLen);
pQueryMsg->udfNum = htonl(pQueryMsg->udfNum);
// query msg safety check
if (!validateQueryMsg(pQueryMsg)) {
@ -6933,6 +7079,33 @@ int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param) {
pMsg += sizeof(int32_t);
}
if (pQueryMsg->udfContentLen > 0) {
param->pUdfInfo = calloc(1, sizeof(SUdfInfo));
param->pUdfInfo->contLen = pQueryMsg->udfContentLen;
pMsg = (char*)pQueryMsg + pQueryMsg->udfContentOffset;
param->pUdfInfo->resType = *(int8_t*) pMsg;
pMsg += sizeof(int8_t);
param->pUdfInfo->resBytes = htons(*(int16_t*)pMsg);
pMsg += sizeof(int16_t);
tstr* name = (tstr*)(pMsg);
param->pUdfInfo->name = strndup(name->data, name->len);
pMsg += varDataTLen(name);
param->pUdfInfo->funcType = htonl(*(int32_t*)pMsg);
pMsg += sizeof(int32_t);
param->pUdfInfo->bufSize = htonl(*(int32_t*)pMsg);
pMsg += sizeof(int32_t);
param->pUdfInfo->content = malloc(pQueryMsg->udfContentLen);
memcpy(param->pUdfInfo->content, pMsg, pQueryMsg->udfContentLen);
pMsg += pQueryMsg->udfContentLen;
}
param->sql = strndup(pMsg, pQueryMsg->sqlstrLen);
SQueriedTableInfo info = { .numOfTags = pQueryMsg->numOfTags, .numOfCols = pQueryMsg->numOfCols, .colList = pQueryMsg->tableCols};
@ -7023,7 +7196,7 @@ static int32_t updateOutputBufForTopBotQuery(SQueriedTableInfo* pTableInfo, SCol
} else {
SColumnInfo* pCol = &pTableInfo->colList[j];
int32_t ret = getResultDataInfo(pCol->type, pCol->bytes, functId, (int32_t)pExprs[i].base.param[0].i64,
&pExprs[i].base.resType, &pExprs[i].base.resBytes, &pExprs[i].base.interBytes, tagLen, superTable);
&pExprs[i].base.resType, &pExprs[i].base.resBytes, &pExprs[i].base.interBytes, tagLen, superTable, NULL);
assert(ret == TSDB_CODE_SUCCESS);
}
}
@ -7032,12 +7205,138 @@ static int32_t updateOutputBufForTopBotQuery(SQueriedTableInfo* pTableInfo, SCol
return TSDB_CODE_SUCCESS;
}
void destroyUdfInfo(SUdfInfo* pUdfInfo) {
if (pUdfInfo == NULL) {
return;
}
if (pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY]) {
if (pUdfInfo->isScript) {
(*(scriptDestroyFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY])(pUdfInfo->pScriptCtx);
tfree(pUdfInfo->content);
}else{
(*(udfDestroyFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY])(&pUdfInfo->init);
}
}
tfree(pUdfInfo->name);
if (pUdfInfo->path) {
unlink(pUdfInfo->path);
}
tfree(pUdfInfo->path);
tfree(pUdfInfo->content);
taosCloseDll(pUdfInfo->handle);
tfree(pUdfInfo);
}
static char* getUdfFuncName(char* name, int type) {
char* funcname = calloc(1, TSDB_FUNCTIONS_NAME_MAX_LENGTH + 10);
switch (type) {
case TSDB_UDF_FUNC_NORMAL:
strcpy(funcname, name);
break;
case TSDB_UDF_FUNC_INIT:
sprintf(funcname, "%s_init", name);
break;
case TSDB_UDF_FUNC_FINALIZE:
sprintf(funcname, "%s_finalize", name);
break;
case TSDB_UDF_FUNC_MERGE:
sprintf(funcname, "%s_merge", name);
break;
case TSDB_UDF_FUNC_DESTROY:
sprintf(funcname, "%s_destroy", name);
break;
default:
assert(0);
break;
}
return funcname;
}
int32_t initUdfInfo(SUdfInfo* pUdfInfo) {
if (pUdfInfo == NULL) {
return TSDB_CODE_SUCCESS;
}
//qError("script len: %d", pUdfInfo->contLen);
if (isValidScript(pUdfInfo->content, pUdfInfo->contLen)) {
pUdfInfo->isScript = 1;
pUdfInfo->pScriptCtx = createScriptCtx(pUdfInfo->content, pUdfInfo->resType, pUdfInfo->resBytes);
if (pUdfInfo->pScriptCtx == NULL) {
return TSDB_CODE_QRY_SYS_ERROR;
}
tfree(pUdfInfo->content);
pUdfInfo->funcs[TSDB_UDF_FUNC_INIT] = taosLoadScriptInit;
if (pUdfInfo->funcs[TSDB_UDF_FUNC_INIT] == NULL
|| (*(scriptInitFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_INIT])(pUdfInfo->pScriptCtx) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_QRY_SYS_ERROR;
}
pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL] = taosLoadScriptNormal;
if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) {
pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE] = taosLoadScriptFinalize;
pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE] = taosLoadScriptMerge;
}
pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY] = taosLoadScriptDestroy;
} else {
char path[PATH_MAX] = {0};
taosGetTmpfilePath("script", path);
FILE* file = fopen(path, "w+");
// TODO check for failure of flush to disk
/*size_t t = */ fwrite(pUdfInfo->content, pUdfInfo->contLen, 1, file);
fclose(file);
tfree(pUdfInfo->content);
pUdfInfo->path = strdup(path);
pUdfInfo->handle = taosLoadDll(path);
if (NULL == pUdfInfo->handle) {
return TSDB_CODE_QRY_SYS_ERROR;
}
pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(pUdfInfo->name, TSDB_UDF_FUNC_NORMAL));
if (NULL == pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL]) {
return TSDB_CODE_QRY_SYS_ERROR;
}
pUdfInfo->funcs[TSDB_UDF_FUNC_INIT] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(pUdfInfo->name, TSDB_UDF_FUNC_INIT));
if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) {
pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(pUdfInfo->name, TSDB_UDF_FUNC_FINALIZE));
pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(pUdfInfo->name, TSDB_UDF_FUNC_MERGE));
}
pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(pUdfInfo->name, TSDB_UDF_FUNC_DESTROY));
if (pUdfInfo->funcs[TSDB_UDF_FUNC_INIT]) {
return (*(udfInitFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_INIT])(&pUdfInfo->init);
}
}
return TSDB_CODE_SUCCESS;
}
// TODO tag length should be passed from client, refactor
int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExprInfo** pExprInfo,
SSqlExpr** pExprMsg, SColumnInfo* pTagCols, int32_t queryType, void* pMsg) {
SSqlExpr** pExprMsg, SColumnInfo* pTagCols, int32_t queryType, void* pMsg, SUdfInfo* pUdfInfo) {
*pExprInfo = NULL;
int32_t code = TSDB_CODE_SUCCESS;
code = initUdfInfo(pUdfInfo);
if (code) {
return code;
}
SExprInfo *pExprs = (SExprInfo *)calloc(numOfOutput, sizeof(SExprInfo));
if (pExprs == NULL) {
return TSDB_CODE_QRY_OUT_OF_MEMORY;
@ -7126,7 +7425,7 @@ int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExp
// todo remove it
if (getResultDataInfo(type, bytes, pExprs[i].base.functionId, param, &pExprs[i].base.resType, &pExprs[i].base.resBytes,
&pExprs[i].base.interBytes, 0, isSuperTable) != TSDB_CODE_SUCCESS) {
&pExprs[i].base.interBytes, 0, isSuperTable, pUdfInfo) != TSDB_CODE_SUCCESS) {
tfree(pExprs);
return TSDB_CODE_QRY_INVALID_MSG;
}
@ -7147,7 +7446,7 @@ int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExp
// todo refactor
int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg* pQueryMsg, int32_t numOfOutput, SExprInfo** pExprInfo,
SSqlExpr** pExpr, SExprInfo* prevExpr) {
SSqlExpr** pExpr, SExprInfo* prevExpr, SUdfInfo *pUdfInfo) {
*pExprInfo = NULL;
int32_t code = TSDB_CODE_SUCCESS;
@ -7192,7 +7491,7 @@ int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg* pQueryMsg, int32_t nu
int32_t param = (int32_t)pExprs[i].base.param[0].i64;
if (getResultDataInfo(type, bytes, pExprs[i].base.functionId, param, &pExprs[i].base.resType, &pExprs[i].base.resBytes,
&pExprs[i].base.interBytes, 0, isSuperTable) != TSDB_CODE_SUCCESS) {
&pExprs[i].base.interBytes, 0, isSuperTable, pUdfInfo) != TSDB_CODE_SUCCESS) {
tfree(pExprs);
return TSDB_CODE_QRY_INVALID_MSG;
}
@ -7378,7 +7677,7 @@ FORCE_INLINE bool checkQIdEqual(void *qHandle, uint64_t qId) {
SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, SExprInfo* pExprs,
SExprInfo* pSecExprs, STableGroupInfo* pTableGroupInfo, SColumnInfo* pTagCols, int32_t vgId,
char* sql, uint64_t qId) {
char* sql, uint64_t qId, SUdfInfo* pUdfInfo) {
int16_t numOfCols = pQueryMsg->numOfCols;
int16_t numOfOutput = pQueryMsg->numOfOutput;
@ -7389,6 +7688,8 @@ SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, S
pQInfo->qId = qId;
pQInfo->runtimeEnv.pUdfInfo = pUdfInfo;
// to make sure third party won't overwrite this structure
pQInfo->signature = pQInfo;
SQueryAttr* pQueryAttr = &pQInfo->query;
@ -7412,6 +7713,7 @@ SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, S
pQueryAttr->prjInfo.vgroupLimit = pQueryMsg->vgroupLimit;
pQueryAttr->prjInfo.ts = (pQueryMsg->order == TSDB_ORDER_ASC)? INT64_MIN:INT64_MAX;
pQueryAttr->sw = pQueryMsg->sw;
pQueryAttr->vgId = vgId;
pQueryAttr->stableQuery = pQueryMsg->stableQuery;
pQueryAttr->topBotQuery = pQueryMsg->topBotQuery;
@ -7444,7 +7746,6 @@ SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, S
}
}
// calculate the result row size
for (int16_t col = 0; col < numOfOutput; ++col) {
assert(pExprs[col].base.resBytes > 0);
pQueryAttr->resultRowSize += pExprs[col].base.resBytes;
@ -7465,6 +7766,20 @@ SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, S
goto _cleanup;
}
if (pSecExprs != NULL) {
int32_t resultRowSize = 0;
// calculate the result row size
for (int16_t col = 0; col < pQueryAttr->numOfExpr2; ++col) {
assert(pSecExprs[col].base.resBytes > 0);
resultRowSize += pSecExprs[col].base.resBytes;
}
if (resultRowSize > pQueryAttr->resultRowSize) {
pQueryAttr->resultRowSize = resultRowSize;
}
}
if (pQueryAttr->fillType != TSDB_FILL_NONE) {
pQueryAttr->fillVal = malloc(sizeof(int64_t) * pQueryAttr->numOfOutput);
if (pQueryAttr->fillVal == NULL) {
@ -7610,8 +7925,8 @@ int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo*
SArray* prevResult = NULL;
if (prevResultLen > 0) {
prevResult = interResFromBinary(param->prevResult, prevResultLen);
prevResult = interResFromBinary(param->prevResult, prevResultLen);
pRuntimeEnv->prevResult = prevResult;
}

443
src/query/src/qScript.c Normal file
View File

@ -0,0 +1,443 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "qScript.h"
#include "ttype.h"
#include "tstrbuild.h"
#include "queryLog.h"
#include "ttokendef.h"
static ScriptEnvPool *pool = NULL;
static ScriptEnv* getScriptEnvFromPool();
static void addScriptEnvToPool(ScriptEnv *pEnv);
static lua_State* createLuaEnv();
static void destroyLuaEnv(lua_State *state);
static void destroyScriptEnv(ScriptEnv *pEnv);
static void luaValueToTaosType(lua_State *lua, char *interBuf, int32_t *numOfOutput, int16_t oType, int16_t oBytes);
static void taosValueToLuaType(lua_State *lua, int32_t type, char *val);
static bool hasBaseFuncDefinedInScript(lua_State *lua, const char *funcPrefix, int32_t len);
static int userlib_exampleFunc(lua_State *lua) {
double op1 = luaL_checknumber(lua,1);
double op2 = luaL_checknumber(lua,2);
lua_pushnumber(lua, op1 * op2);
return 1;
}
void luaRegisterLibFunc(lua_State *lua) {
lua_register(lua, "exampleFunc", userlib_exampleFunc);
}
void luaLoadLib(lua_State *lua, const char *libname, lua_CFunction luafunc) {
lua_pushcfunction(lua, luafunc);
lua_pushstring(lua, libname);
lua_call(lua, 1, 0);
}
LUALIB_API int (luaopen_cjson) (lua_State *L);
LUALIB_API int (luaopen_struct) (lua_State *L);
LUALIB_API int (luaopen_cmsgpack) (lua_State *L);
LUALIB_API int (luaopen_bit) (lua_State *L);
static void luaLoadLibraries(lua_State *lua) {
luaLoadLib(lua, "", luaopen_base);
luaLoadLib(lua, LUA_TABLIBNAME, luaopen_table);
luaLoadLib(lua, LUA_STRLIBNAME, luaopen_string);
luaLoadLib(lua, LUA_MATHLIBNAME, luaopen_math);
luaLoadLib(lua, LUA_DBLIBNAME, luaopen_debug);
}
static void luaRemoveUnsupportedFunctions(lua_State *lua) {
lua_pushnil(lua);
lua_setglobal(lua,"loadfile");
lua_pushnil(lua);
lua_setglobal(lua,"dofile");
}
void taosValueToLuaType(lua_State *lua, int32_t type, char *val) {
//TODO(dengyihao): handle more data type
if (IS_SIGNED_NUMERIC_TYPE(type) || type == TSDB_DATA_TYPE_BOOL) {
int64_t v;
GET_TYPED_DATA(v, int64_t, type, val);
lua_pushnumber(lua, (lua_Number)v);
} else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
uint64_t v;
GET_TYPED_DATA(v, uint64_t, type, val);
lua_pushnumber(lua,(lua_Number)v);
} else if (IS_FLOAT_TYPE(type)) {
double v;
GET_TYPED_DATA(v, double, type, val);
lua_pushnumber(lua,v);
} else if (type == TSDB_DATA_TYPE_BINARY) {
lua_pushlstring(lua, val, varDataLen(val));
} else if (type == TSDB_DATA_TYPE_NCHAR) {
}
}
int taosLoadScriptInit(void* pInit) {
ScriptCtx *pCtx = pInit;
char funcName[MAX_FUNC_NAME] = {0};
sprintf(funcName, "%s_init", pCtx->funcName);
lua_State* lua = pCtx->pEnv->lua_state;
lua_getglobal(lua, funcName);
if (lua_pcall(lua, 0, -1, 0)) {
lua_pop(lua, -1);
}
lua_setglobal(lua, "global");
return 0;
}
void taosLoadScriptNormal(void *pInit, char *pInput, int16_t iType, int16_t iBytes, int32_t numOfRows,
int64_t *ptsList, int64_t key, char* pOutput, char *ptsOutput, int32_t *numOfOutput, int16_t oType, int16_t oBytes) {
ScriptCtx* pCtx = pInit;
char funcName[MAX_FUNC_NAME] = {0};
sprintf(funcName, "%s_add", pCtx->funcName);
lua_State* lua = pCtx->pEnv->lua_state;
lua_getglobal(lua, funcName);
// first param of script;
lua_newtable(lua);
int32_t offset = 0;
for (int32_t i = 0; i < numOfRows; i++) {
taosValueToLuaType(lua, iType, pInput + offset);
lua_rawseti(lua, -2, i+1);
offset += iBytes;
}
int isGlobalState = false;
lua_getglobal(lua, "global");
if (lua_istable(lua, -1)) {
isGlobalState = true;
}
lua_pushnumber(lua, (lua_Number)key);
// do call lua script
if (lua_pcall(lua, 3, 1, 0) != 0) {
qError("SCRIPT ERROR: %s", lua_tostring(lua, -1));
lua_pop(lua, -1);
return;
}
int tNumOfOutput = 0;
if (isGlobalState == false) {
luaValueToTaosType(lua, pOutput, &tNumOfOutput, oType, oBytes);
} else {
lua_setglobal(lua, "global");
}
*numOfOutput = tNumOfOutput;
}
void taosLoadScriptMerge(void *pInit, char* data, int32_t numOfRows, char* pOutput, int32_t* numOfOutput) {
ScriptCtx *pCtx = pInit;
char funcName[MAX_FUNC_NAME] = {0};
sprintf(funcName, "%s_merge", pCtx->funcName);
lua_State* lua = pCtx->pEnv->lua_state;
lua_getglobal(lua, funcName);
if (!lua_isfunction(lua, -1)) {
qError("SCRIPT ERROR: %s", lua_tostring(lua, -1));
return;
}
lua_getglobal(lua, "global");
if (lua_pcall(lua, 1, 1, 0) != 0) {
qError("SCRIPT ERROR: %s", lua_tostring(lua, -1));
lua_pop(lua, -1);
return;
}
int tNumOfOutput = 0;
luaValueToTaosType(lua, pOutput, &tNumOfOutput, pCtx->resType, pCtx->resBytes);
*numOfOutput = tNumOfOutput;
}
//do not support agg now
void taosLoadScriptFinalize(void *pInit,int64_t key, char *pOutput, int32_t* numOfOutput) {
ScriptCtx *pCtx = pInit;
char funcName[MAX_FUNC_NAME] = {0};
sprintf(funcName, "%s_finalize", pCtx->funcName);
lua_State* lua = pCtx->pEnv->lua_state;
lua_getglobal(lua, funcName);
if (!lua_isfunction(lua, -1)) {
qError("SCRIPT ERROR: %s", lua_tostring(lua, -1));
return;
}
lua_getglobal(lua, "global");
lua_pushnumber(lua, (lua_Number)key);
if (lua_pcall(lua, 2, 2, 0) != 0) {
qError("SCRIPT ERROR: %s", lua_tostring(lua, -1));
lua_pop(lua, -1);
return;
}
lua_setglobal(lua, "global");
int tNumOfOutput = 0;
luaValueToTaosType(lua, pOutput, &tNumOfOutput, pCtx->resType, pCtx->resBytes);
*numOfOutput = tNumOfOutput;
}
void taosLoadScriptDestroy(void *pInit) {
destroyScriptCtx(pInit);
}
ScriptCtx* createScriptCtx(char *script, int8_t resType, int16_t resBytes) {
ScriptCtx *pCtx = (ScriptCtx *)calloc(1, sizeof(ScriptCtx));
pCtx->state = SCRIPT_STATE_INIT;
pCtx->pEnv = getScriptEnvFromPool(); //
pCtx->resType = resType;
pCtx->resBytes = resBytes;
if (pCtx->pEnv == NULL) {
destroyScriptCtx(pCtx);
return NULL;
}
lua_State *lua = pCtx->pEnv->lua_state;
if (luaL_dostring(lua, script)) {
lua_pop(lua, 1);
qError("dynamic load script failed");
destroyScriptCtx(pCtx);
return NULL;
}
lua_getglobal(lua, USER_FUNC_NAME);
const char *name = lua_tostring(lua, -1);
if (name == NULL) {
lua_pop(lua, 1);
qError("SCRIPT ERROR: invalid script");
destroyScriptCtx(pCtx);
return NULL;
}
memcpy(pCtx->funcName, name, strlen(name));
lua_pop(lua, 1);
return pCtx;
}
void destroyScriptCtx(void *pCtx) {
if (pCtx == NULL) return;
addScriptEnvToPool(((ScriptCtx *)pCtx)->pEnv);
free(pCtx);
}
void luaValueToTaosType(lua_State *lua, char *interBuf, int32_t *numOfOutput, int16_t oType, int16_t oBytes) {
int t = lua_type(lua,-1);
int32_t sz = 0;
switch (t) {
case LUA_TSTRING:
//TODO(yihaodeng): handle str type
{
const char *v = lua_tostring(lua, -1);
memcpy(interBuf, v, strlen(v));
sz = 1;
}
break;
case LUA_TBOOLEAN:
{
double v = lua_tonumber(lua, -1);
memcpy(interBuf, (char *)&v, oBytes);
sz = 1;
}
break;
case LUA_TNUMBER:
{
if (oType == TSDB_DATA_TYPE_FLOAT) {
float v = (float)lua_tonumber(lua, -1);
memcpy(interBuf, (char *)&v, oBytes);
sz = 1;
} else if (oType == TSDB_DATA_TYPE_DOUBLE) {
double v = (double)lua_tonumber(lua, -1);
memcpy(interBuf, (char *)&v, oBytes);
sz = 1;
} else if (oType == TSDB_DATA_TYPE_BIGINT) {
int64_t v = (int64_t)lua_tonumber(lua, -1);
memcpy(interBuf, (char *)&v, oBytes);
sz = 1;
} else if (oType <= TSDB_DATA_TYPE_INT) {
int32_t v = (int32_t)lua_tonumber(lua, -1);
memcpy(interBuf, (char *)&v, oBytes);
sz = 1;
}
}
break;
case LUA_TTABLE:
{
lua_pushnil(lua);
int32_t offset = 0;
while(lua_next(lua, -2)) {
int32_t v = (int32_t)lua_tonumber(lua, -1);
memcpy(interBuf + offset, (char *)&v, oBytes);
offset += oBytes;
lua_pop(lua, 1);
sz += 1;
}
}
break;
default:
setNull(interBuf, oType, oBytes);
sz = 1;
break;
}
lua_pop(lua,1); // pop ret value from script
*numOfOutput = sz;
}
/*
*Initialize the scripting environment.
*/
lua_State* createLuaEnv() {
lua_State *lua = lua_open();
luaLoadLibraries(lua);
luaRemoveUnsupportedFunctions(lua);
// register func in external lib
luaRegisterLibFunc(lua);
{
char *errh_func = "local dbg = debug\n"
"function __taos__err__handler(err)\n"
" local i = dbg.getinfo(2,'nSl')\n"
" if i and i.what == 'C' then\n"
" i = dbg.getinfo(3,'nSl')\n"
" end\n"
" if i then\n"
" return i.source .. ':' .. i.currentline .. ': ' .. err\n"
" else\n"
" return err\n"
" end\n"
"end\n";
luaL_loadbuffer(lua,errh_func,strlen(errh_func),"@err_handler_def");
lua_pcall(lua,0,0,0);
}
return lua;
}
void destroyLuaEnv(lua_State *lua) {
lua_close(lua);
}
int32_t scriptEnvPoolInit() {
const int size = 10; // configure or not
pool = malloc(sizeof(ScriptEnvPool));
pthread_mutex_init(&pool->mutex, NULL);
pool->scriptEnvs = tdListNew(sizeof(ScriptEnv *));
for (int i = 0; i < size; i++) {
ScriptEnv *env = malloc(sizeof(ScriptEnv));
env->funcId = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);;
env->lua_state = createLuaEnv();
tdListAppend(pool->scriptEnvs, (void *)(&env));
}
pool->mSize = size;
pool->cSize = size;
return 0;
}
void scriptEnvPoolCleanup() {
if (pool == NULL) return;
SListNode *pNode = NULL;
while ((pNode = tdListPopHead(pool->scriptEnvs)) != NULL) {
ScriptEnv *pEnv = NULL;
tdListNodeGetData(pool->scriptEnvs, pNode, (void *)(&pEnv));
destroyScriptEnv(pEnv);
listNodeFree(pNode);
}
tdListFree(pool->scriptEnvs);
pthread_mutex_destroy(&pool->mutex);
free(pool);
}
void destroyScriptEnv(ScriptEnv *pEnv) {
destroyLuaEnv(pEnv->lua_state);
taosHashCleanup(pEnv->funcId);
free(pEnv);
}
ScriptEnv* getScriptEnvFromPool() {
ScriptEnv *pEnv = NULL;
pthread_mutex_lock(&pool->mutex);
if (pool->cSize <= 0) {
pthread_mutex_unlock(&pool->mutex);
return NULL;
}
SListNode *pNode = tdListPopHead(pool->scriptEnvs);
tdListNodeGetData(pool->scriptEnvs, pNode, (void *)(&pEnv));
listNodeFree(pNode);
pool->cSize--;
pthread_mutex_unlock(&pool->mutex);
return pEnv;
}
void addScriptEnvToPool(ScriptEnv *pEnv) {
if (pEnv == NULL) {
return;
}
pthread_mutex_lock(&pool->mutex);
lua_settop(pEnv->lua_state, 0);
tdListAppend(pool->scriptEnvs, (void *)(&pEnv));
pool->cSize++;
pthread_mutex_unlock(&pool->mutex);
}
bool hasBaseFuncDefinedInScript(lua_State *lua, const char *funcPrefix, int32_t len) {
bool ret = true;
char funcName[MAX_FUNC_NAME];
memcpy(funcName, funcPrefix, len);
const char *base[] = {"_init", "_add"};
for (int i = 0; (i < sizeof(base)/sizeof(base[0])) && (ret == true); i++) {
memcpy(funcName + len, base[i], strlen(base[i]));
memset(funcName + len + strlen(base[i]), 0, MAX_FUNC_NAME - len - strlen(base[i]));
lua_getglobal(lua, funcName);
ret = lua_isfunction(lua, -1); // exsit function or not
lua_pop(lua, 1);
}
return ret;
}
bool isValidScript(char *script, int32_t len) {
ScriptEnv *pEnv = getScriptEnvFromPool(); //
if (pEnv == NULL) {
return false;
}
lua_State *lua = pEnv->lua_state;
if (len < strlen(script)) {
script[len] = 0;
}
if (luaL_dostring(lua, script)) {
lua_pop(lua, 1);
addScriptEnvToPool(pEnv);
qError("error at %s and %d", script, (int)(strlen(script)));
return false;
}
lua_getglobal(lua, USER_FUNC_NAME);
const char *name = lua_tostring(lua, -1);
if (name == NULL || strlen(name) >= USER_FUNC_NAME_LIMIT) {
lua_pop(lua, 1);
addScriptEnvToPool(pEnv);
qError("error at %s name: %s, len = %d", script, name, (int)(strlen(name)));
return false;
}
bool ret = hasBaseFuncDefinedInScript(lua, name, (int32_t)strlen(name));
lua_pop(lua, 1); // pop
addScriptEnvToPool(pEnv);
return ret;
}

48
src/query/src/qSqlParser.c
View File

@ -28,6 +28,7 @@ SSqlInfo qSqlParse(const char *pStr) {
SSqlInfo sqlInfo = {0};
sqlInfo.valid = true;
sqlInfo.funcs = taosArrayInit(4, sizeof(SStrToken));
int32_t i = 0;
while (1) {
@ -120,6 +121,19 @@ void tSqlExprListDestroy(SArray *pList) {
taosArrayDestroyEx(pList, freeExprElem);
}
SArray *tStrTokenAppend(SArray *pList, SStrToken *pToken) {
if (pList == NULL) {
pList = taosArrayInit(4, sizeof(tVariantListItem));
}
if (pToken) {
taosArrayPush(pList, pToken);
}
return pList;
}
tSqlExpr *tSqlExprCreateIdValue(SStrToken *pToken, int32_t optrType) {
tSqlExpr *pSqlExpr = calloc(1, sizeof(tSqlExpr));
@ -922,8 +936,8 @@ void* destroyCreateTableSql(SCreateTableSql* pCreate) {
}
void SqlInfoDestroy(SSqlInfo *pInfo) {
if (pInfo == NULL) return;
if (pInfo == NULL) return;;
taosArrayDestroy(pInfo->funcs);
if (pInfo->type == TSDB_SQL_SELECT) {
destroyAllSqlNode(pInfo->list);
} else if (pInfo->type == TSDB_SQL_CREATE_TABLE) {
@ -1018,6 +1032,18 @@ void setDropDbTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrTo
pInfo->pMiscInfo->tableType = tableType;
}
void setDropFuncInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken) {
pInfo->type = type;
if (pInfo->pMiscInfo == NULL) {
pInfo->pMiscInfo = (SMiscInfo *)calloc(1, sizeof(SMiscInfo));
pInfo->pMiscInfo->a = taosArrayInit(4, sizeof(SStrToken));
}
taosArrayPush(pInfo->pMiscInfo->a, pToken);
}
void setShowOptions(SSqlInfo *pInfo, int32_t type, SStrToken* prefix, SStrToken* pPatterns) {
if (pInfo->pMiscInfo == NULL) {
pInfo->pMiscInfo = calloc(1, sizeof(SMiscInfo));
@ -1052,6 +1078,24 @@ void setCreateDbInfo(SSqlInfo *pInfo, int32_t type, SStrToken *pToken, SCreateDb
pInfo->pMiscInfo->dbOpt.ignoreExists = pIgExists->n; // sql.y has: ifnotexists(X) ::= IF NOT EXISTS. {X.n = 1;}
}
void setCreateFuncInfo(SSqlInfo *pInfo, int32_t type, SStrToken *pName, SStrToken *pPath, TAOS_FIELD *output, SStrToken* bufSize, int32_t funcType) {
pInfo->type = type;
if (pInfo->pMiscInfo == NULL) {
pInfo->pMiscInfo = calloc(1, sizeof(SMiscInfo));
}
pInfo->pMiscInfo->funcOpt.name = *pName;
pInfo->pMiscInfo->funcOpt.path = *pPath;
pInfo->pMiscInfo->funcOpt.output = *output;
pInfo->pMiscInfo->funcOpt.type = funcType;
if (bufSize->n > 0) {
pInfo->pMiscInfo->funcOpt.bufSize = strtol(bufSize->z, NULL, 10);
} else {
pInfo->pMiscInfo->funcOpt.bufSize = 0;
}
}
void setCreateAcctSql(SSqlInfo *pInfo, int32_t type, SStrToken *pName, SStrToken *pPwd, SCreateAcctInfo *pAcctInfo) {
pInfo->type = type;
if (pInfo->pMiscInfo == NULL) {

16
src/query/src/queryMain.c
View File

@ -68,7 +68,7 @@ void freeParam(SQueryParam *param) {
tfree(param->prevResult);
}
int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qinfo_t* pQInfo, uint64_t *qId) {
int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qinfo_t* pQInfo, uint64_t qId) {
assert(pQueryMsg != NULL && tsdb != NULL);
int32_t code = TSDB_CODE_SUCCESS;
@ -93,12 +93,12 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi
SQueriedTableInfo info = { .numOfTags = pQueryMsg->numOfTags, .numOfCols = pQueryMsg->numOfCols, .colList = pQueryMsg->tableCols};
if ((code = createQueryFunc(&info, pQueryMsg->numOfOutput, &param.pExprs, param.pExpr, param.pTagColumnInfo,
pQueryMsg->queryType, pQueryMsg)) != TSDB_CODE_SUCCESS) {
pQueryMsg->queryType, pQueryMsg, param.pUdfInfo)) != TSDB_CODE_SUCCESS) {
goto _over;
}
if (param.pSecExpr != NULL) {
if ((code = createIndirectQueryFuncExprFromMsg(pQueryMsg, pQueryMsg->secondStageOutput, &param.pSecExprs, param.pSecExpr, param.pExprs)) != TSDB_CODE_SUCCESS) {
if ((code = createIndirectQueryFuncExprFromMsg(pQueryMsg, pQueryMsg->secondStageOutput, &param.pSecExprs, param.pSecExpr, param.pExprs, param.pUdfInfo)) != TSDB_CODE_SUCCESS) {
goto _over;
}
}
@ -162,18 +162,14 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi
assert(pQueryMsg->stableQuery == isSTableQuery);
(*pQInfo) = createQInfoImpl(pQueryMsg, param.pGroupbyExpr, param.pExprs, param.pSecExprs, &tableGroupInfo,
param.pTagColumnInfo, vgId, param.sql, *qId);
param.pTagColumnInfo, vgId, param.sql, qId, param.pUdfInfo);
param.sql = NULL;
param.pExprs = NULL;
param.pSecExprs = NULL;
param.pGroupbyExpr = NULL;
param.pTagColumnInfo = NULL;
if ((*pQInfo) == NULL) {
code = TSDB_CODE_QRY_OUT_OF_MEMORY;
goto _over;
}
param.pUdfInfo = NULL;
code = initQInfo(&pQueryMsg->tsBuf, tsdb, NULL, *pQInfo, &param, (char*)pQueryMsg, pQueryMsg->prevResultLen, NULL);
@ -182,6 +178,8 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi
taosArrayDestroy(param.pGroupbyExpr->columnInfo);
}
destroyUdfInfo(param.pUdfInfo);
taosArrayDestroy(param.pTableIdList);
param.pTableIdList = NULL;

3304
src/query/src/sql.c
View File

File diff suppressed because it is too large Load Diff

11
src/util/src/terror.c
View File

@ -109,11 +109,12 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_SQL_SYNTAX_ERROR, "Syntax error in SQL")
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DB_NOT_SELECTED, "Database not specified or available")
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TABLE_NAME, "Table does not exist")
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_EXCEED_SQL_LIMIT, "SQL statement too long, check maxSQLLength config")
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_FILE_EMPTY, "File is empty")
// mnode
TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED, "Message not processed")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACTION_IN_PROGRESS, "Message is progressing")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACTION_NEED_REPROCESSED, "Messag need to be reprocessed")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACTION_NEED_REPROCESSED, "Message need to be reprocessed")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_RIGHTS, "Insufficient privilege for operation")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_APP_ERROR, "Unexpected generic error in mnode")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CONNECTION, "Invalid message connection")
@ -183,6 +184,13 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_NOT_EXIST, "Field does not exist"
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STABLE_NAME, "Super table does not exist")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CREATE_TABLE_MSG, "Invalid create table message")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_NAME, "Invalid func name")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_LEN, "Invalid func length")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_CODE, "Invalid func code")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_FUNC_ALREADY_EXIST, "Func already exists")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC, "Invalid func")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_BUFSIZE, "Invalid func bufSize")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_NOT_SELECTED, "Database not specified or available")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_ALREADY_EXIST, "Database already exists")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION, "Invalid database options")
@ -269,6 +277,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_IN_EXEC, "Multiple retrieval of
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, "Too many time window in query")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_ENOUGH_BUFFER, "Query buffer limit has reached")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INCONSISTAN, "File inconsistance in replica")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_SYS_ERROR, "System error")
// grant

1
src/util/src/tskiplist.c
View File

@ -546,7 +546,6 @@ static FORCE_INLINE int32_t getSkipListNodeRandomHeight(SSkipList *pSkipList) {
const uint32_t factor = 4;
int32_t n = 1;
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
while ((rand() % factor) == 0 && n <= pSkipList->maxLevel) {
#else

11
src/util/src/ttokenizer.c
View File

@ -219,8 +219,13 @@ static SKeyword keywordTable[] = {
{"PARTITIONS", TK_PARTITIONS},
{"TOPIC", TK_TOPIC},
{"TOPICS", TK_TOPICS},
{"COMPACT", TK_COMPACT},
{"MODIFY", TK_MODIFY}
{"COMPACT", TK_COMPACT},
{"MODIFY", TK_MODIFY},
{"FUNCTION", TK_FUNCTION},
{"FUNCTIONS", TK_FUNCTIONS},
{"OUTPUTTYPE", TK_OUTPUTTYPE},
{"AGGREGATE", TK_AGGREGATE},
{"BUFSIZE", TK_BUFSIZE},
};
static const char isIdChar[] = {
@ -269,7 +274,7 @@ static int32_t tKeywordCode(const char* z, int n) {
if (keywordHashTable == NULL) {
return TK_ILLEGAL;
}
SKeyword** pKey = (SKeyword**)taosHashGet(keywordHashTable, key, n);
return (pKey != NULL)? (*pKey)->type:TK_ID;
}

2
src/vnode/src/vnodeRead.c
View File

@ -231,7 +231,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
if (contLen != 0) {
qinfo_t pQInfo = NULL;
uint64_t qId = genQueryId();
code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, &pQInfo, &qId);
code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, &pQInfo, qId);
SQueryTableRsp *pRsp = (SQueryTableRsp *)rpcMallocCont(sizeof(SQueryTableRsp));
pRsp->code = code;

6
tests/examples/c/CMakeLists.txt
View File

@ -8,13 +8,13 @@ IF (TD_LINUX)
ADD_EXECUTABLE(subscribe subscribe.c)
TARGET_LINK_LIBRARIES(subscribe taos_static trpc tutil pthread )
ADD_EXECUTABLE(epoll epoll.c)
TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread )
TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread lua)
ENDIF ()
IF (TD_DARWIN)
INCLUDE_DIRECTORIES(. ${TD_COMMUNITY_DIR}/src/inc ${TD_COMMUNITY_DIR}/src/client/inc ${TD_COMMUNITY_DIR}/inc)
AUX_SOURCE_DIRECTORY(. SRC)
ADD_EXECUTABLE(demo demo.c)
TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread )
TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread lua)
ADD_EXECUTABLE(epoll epoll.c)
TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread )
TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread lua)
ENDIF ()

7
tests/script/general/parser/testSuite.sim
View File

@ -12,7 +12,7 @@ run general/parser/create_tb.sim
run general/parser/dbtbnameValidate.sim
run general/parser/fill.sim
run general/parser/fill_stb.sim
run general/parser/fill_us.sim
#run general/parser/fill_us.sim #
run general/parser/first_last.sim
run general/parser/import_commit1.sim
run general/parser/import_commit2.sim
@ -39,7 +39,6 @@ run general/parser/slimit1.sim
run general/parser/slimit_alter_tags.sim
run general/parser/tbnameIn.sim
run general/parser/join.sim
#run general/parser/join_multitables.sim
run general/parser/join_multivnode.sim
run general/parser/join_manyblocks.sim
run general/parser/projection_limit_offset.sim
@ -61,5 +60,9 @@ run general/parser/slimit_alter_tags.sim
run general/parser/binary_escapeCharacter.sim
run general/parser/between_and.sim
run general/parser/last_cache.sim
run general/parser/slimit_alter_tags.sim
run general/parser/udf.sim
run general/parser/udf_dll.sim
run general/parser/udf_dll_stable.sim
run general/parser/nestquery.sim
run general/parser/precision_ns.sim

637
tests/script/general/parser/udf.sim Normal file
View File

@ -0,0 +1,637 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c walLevel -v 1
system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2
system sh/exec.sh -n dnode1 -s start
system sh/prepare_udf.sh
sleep 100
sql connect
print ======================== dnode1 start
sql create function n01 as '/tmp/normal' outputtype int;
sql show functions;
if $rows != 1 then
return -1
endi
if $data00 != n01 then
return -1
endi
if $data01 != /tmp/normal then
return -1
endi
if $data02 != 0 then
return -1
endi
if $data03 != INT then
return -1
endi
if $data05 != 5 then
return -1
endi
sql drop function n01;
sql show functions;
if $rows != 0 then
return -1
endi
sql create function n02 as '/tmp/normal' outputtype bool;
sql show functions;
if $rows != 1 then
return -1
endi
if $data00 != n02 then
return -1
endi
if $data01 != /tmp/normal then
return -1
endi
if $data02 != 0 then
return -1
endi
if $data03 != BOOL then
return -1
endi
if $data05 != 5 then
return -1
endi
sql drop function n02;
sql show functions;
if $rows != 0 then
return -1
endi
sql create function n03 as '/tmp/normal' outputtype TINYINT;
sql show functions;
if $rows != 1 then
return -1
endi
if $data00 != n03 then
return -1
endi
if $data01 != /tmp/normal then
return -1
endi
if $data02 != 0 then
return -1
endi
if $data03 != TINYINT then
return -1
endi
if $data05 != 5 then
return -1
endi
sql drop function n03;
sql show functions;
if $rows != 0 then
return -1
endi
sql create function n04 as '/tmp/normal' outputtype SMALLINT;
sql show functions;
if $rows != 1 then
return -1
endi
if $data00 != n04 then
return -1
endi
if $data01 != /tmp/normal then
return -1
endi
if $data02 != 0 then
return -1
endi
if $data03 != SMALLINT then
return -1
endi
if $data05 != 5 then
return -1
endi
sql drop function n04;
sql show functions;
if $rows != 0 then
return -1
endi
sql create function n05 as '/tmp/normal' outputtype INT;
sql show functions;
if $rows != 1 then
return -1
endi
if $data00 != n05 then
return -1
endi
if $data01 != /tmp/normal then
return -1
endi
if $data02 != 0 then
return -1
endi
if $data03 != INT then
return -1
endi
if $data05 != 5 then
return -1
endi
sql drop function n05;
sql show functions;
if $rows != 0 then
return -1
endi
sql create function n06 as '/tmp/normal' outputtype BIGINT;
sql show functions;
if $rows != 1 then
return -1
endi
if $data00 != n06 then
return -1
endi
if $data01 != /tmp/normal then
return -1
endi
if $data02 != 0 then
return -1
endi
if $data03 != BIGINT then
return -1
endi
if $data05 != 5 then
return -1
endi
sql drop function n06;
sql show functions;
if $rows != 0 then
return -1
endi
sql create function n07 as '/tmp/normal' outputtype FLOAT;
sql show functions;
if $rows != 1 then
return -1
endi
if $data00 != n07 then
return -1
endi
if $data01 != /tmp/normal then
return -1
endi
if $data02 != 0 then
return -1
endi
if $data03 != FLOAT then
return -1
endi
if $data05 != 5 then
return -1
endi
sql drop function n07;
sql show functions;
if $rows != 0 then
return -1
endi
sql create function n08 as '/tmp/normal' outputtype DOUBLE;
sql show functions;
if $rows != 1 then
return -1
endi
if $data00 != n08 then
return -1
endi
if $data01 != /tmp/normal then
return -1
endi
if $data02 != 0 then
return -1
endi
if $data03 != DOUBLE then
return -1
endi
if $data05 != 5 then
return -1
endi
sql drop function n08;
sql show functions;
if $rows != 0 then
return -1
endi
sql create function n09 as '/tmp/normal' outputtype BINARY;
sql show functions;
if $rows != 1 then
return -1
endi
if $data00 != n09 then
return -1
endi
if $data01 != /tmp/normal then
return -1
endi
if $data02 != 0 then
return -1
endi
if $data03 != BINARY(0) then
return -1
endi
if $data05 != 5 then
return -1
endi
sql drop function n09;
sql show functions;
if $rows != 0 then
return -1
endi
sql create function n10 as '/tmp/normal' outputtype BINARY(10);
sql show functions;
if $rows != 1 then
return -1
endi
if $data00 != n10 then
return -1
endi
if $data01 != /tmp/normal then
return -1
endi
if $data02 != 0 then
return -1
endi
if $data03 != BINARY(10) then
return -1
endi
if $data05 != 5 then
return -1
endi
sql drop function n10;
sql show functions;
if $rows != 0 then
return -1
endi
sql create function n11 as '/tmp/normal' outputtype TIMESTAMP;
sql show functions;
if $rows != 1 then
return -1
endi
if $data00 != n11 then
return -1
endi
if $data01 != /tmp/normal then
return -1
endi
if $data02 != 0 then
return -1
endi
if $data03 != TIMESTAMP then
return -1
endi
if $data05 != 5 then
return -1
endi
sql drop function n11;
sql show functions;
if $rows != 0 then
return -1
endi
sql create function n12 as '/tmp/normal' outputtype NCHAR;
sql show functions;
if $rows != 1 then
return -1
endi
if $data00 != n12 then
return -1
endi
if $data01 != /tmp/normal then
return -1
endi
if $data02 != 0 then
return -1
endi
if $data03 != NCHAR(0) then
return -1
endi
if $data05 != 5 then
return -1
endi
sql drop function n12;
sql show functions;
if $rows != 0 then
return -1
endi
sql create function n13 as '/tmp/normal' outputtype NCHAR(10);
sql show functions;
if $rows != 1 then
return -1
endi
if $data00 != n13 then
return -1
endi
if $data01 != /tmp/normal then
return -1
endi
if $data02 != 0 then
return -1
endi
if $data03 != NCHAR(10) then
return -1
endi
if $data05 != 5 then
return -1
endi
sql drop function n13;
sql show functions;
if $rows != 0 then
return -1
endi
sql create function n14 as '/tmp/normal' outputtype TINYINT UNSIGNED;
sql show functions;
if $rows != 1 then
return -1
endi
if $data00 != n14 then
return -1
endi
if $data01 != /tmp/normal then
return -1
endi
if $data02 != 0 then
return -1
endi
if $data05 != 5 then
return -1
endi
sql drop function n14;
sql show functions;
if $rows != 0 then
return -1
endi
sql create function n15 as '/tmp/normal' outputtype SMALLINT UNSIGNED;
sql show functions;
if $rows != 1 then
return -1
endi
if $data00 != n15 then
return -1
endi
if $data01 != /tmp/normal then
return -1
endi
if $data02 != 0 then
return -1
endi
if $data05 != 5 then
return -1
endi
sql drop function n15;
sql show functions;
if $rows != 0 then
return -1
endi
sql create function n16 as '/tmp/normal' outputtype INT UNSIGNED;
sql show functions;
if $rows != 1 then
return -1
endi
if $data00 != n16 then
return -1
endi
if $data01 != /tmp/normal then
return -1
endi
if $data02 != 0 then
return -1
endi
if $data05 != 5 then
return -1
endi
sql drop function n16;
sql show functions;
if $rows != 0 then
return -1
endi
sql create function n17 as '/tmp/normal' outputtype BIGINT UNSIGNED;
sql show functions;
if $rows != 1 then
return -1
endi
if $data00 != n17 then
return -1
endi
if $data01 != /tmp/normal then
return -1
endi
if $data02 != 0 then
return -1
endi
if $data05 != 5 then
return -1
endi
sql drop function n17;
sql show functions;
if $rows != 0 then
return -1
endi
sql create aggregate function n18 as '/tmp/normal' outputtype BIGINT UNSIGNED;
sql show functions;
if $rows != 1 then
return -1
endi
if $data00 != n18 then
return -1
endi
if $data01 != /tmp/normal then
return -1
endi
if $data02 != 1 then
return -1
endi
if $data05 != 5 then
return -1
endi
sql drop function n18;
sql show functions;
if $rows != 0 then
return -1
endi
sql create function t01 as '/tmp/normal' outputtype INT;
sql_error create function t01 as '/tmp/normal' outputtype SMALLINT;
sql drop function t01;
sql create function t01 as '/tmp/normal' outputtype INT;
sql create function t02 as '/tmp/normal' outputtype SMALLINT;
sql show functions;
if $rows != 2 then
return -1
endi
sql_error create function e1 as '/tmp/normal';
sql_error create function e2 as '/tmp/normal' outputtype;
sql_error create function e3 as '/tmp/normal' a;
sql_error create function e4 as '/tmp/normal' outputtype a;
sql_error create function e5 as '/tmp/normal' outputtype bool int;
sql_error create function as '/tmp/normal' outputtype;
sql_error create function e6 as '/tmp/empty' outputtype int;
sql_error create function e7 as '/tmp/big' outputtype int;
sql_error create function e8 as '/tmp/noexistfile' outputtype int;
sql_error create function e0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456 as '/tmp/normal' outputtype int;
sql_error create function e9 as outputtype int;
system sh/exec.sh -n dnode1 -s stop -x SIGINT

494
tests/script/general/parser/udf_dll.sim Normal file
View File

@ -0,0 +1,494 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c walLevel -v 1
system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2
system sh/exec.sh -n dnode1 -s start
system sh/prepare_udf.sh
sleep 100
sql connect
print ======================== dnode1 start
sql create function add_one as '/tmp/add_one.so' outputtype int;
sql create aggregate function sum_double as '/tmp/sum_double.so' outputtype int;
sql show functions;
if $rows != 2 then
return -1
endi
sql create database db;
sql use db;
sql create table tb1 (ts timestamp, f1 int, f2 bool, f3 binary(10));
sql insert into tb1 values ('2021-03-23 17:17:19.660', 1, true, 'tb1-1');
sql insert into tb1 values ('2021-03-23 19:23:28.595', 2, false, 'tb1-2');
sql insert into tb1 values ('2021-03-23 19:33:39.070', 3, true, 'tb1-3');
sql insert into tb1 values ('2021-03-23 19:34:37.670', 4, false, 'tb1-4');
sql insert into tb1 values ('2021-03-24 19:08:06.609', 5, true, 'tb1-5');
sql insert into tb1 values ('2021-03-24 19:26:38.231', 6, false, 'tb1-6');
sql insert into tb1 values ('2021-03-25 10:03:17.688', 7, true, 'tb1-7');
sql select add_one(f1) from tb1;
if $rows != 7 then
return -1
endi
if $data00 != 2 then
return -1
endi
if $data10 != 3 then
return -1
endi
if $data20 != 4 then
return -1
endi
if $data30 != 5 then
return -1
endi
if $data40 != 6 then
return -1
endi
if $data50 != 7 then
return -1
endi
if $data60 != 8 then
return -1
endi
sql select sum_double(f1) from tb1;
if $rows != 1 then
return -1
endi
if $data00 != 56 then
return -1
endi
sql select ts,add_one(f1),f1 from tb1;
if $rows != 7 then
return -1
endi
if $data00 != @21-03-23 17:17:19.660@ then
return -1
endi
if $data01 != 2 then
return -1
endi
if $data02 != 1 then
return -1
endi
if $data10 != @21-03-23 19:23:28.595@ then
return -1
endi
if $data11 != 3 then
return -1
endi
if $data12 != 2 then
return -1
endi
if $data20 != @21-03-23 19:33:39.070@ then
return -1
endi
if $data21 != 4 then
return -1
endi
if $data22 != 3 then
return -1
endi
if $data30 != @21-03-23 19:34:37.670@ then
return -1
endi
if $data31 != 5 then
return -1
endi
if $data32 != 4 then
return -1
endi
if $data40 != @21-03-24 19:08:06.609@ then
return -1
endi
if $data41 != 6 then
return -1
endi
if $data42 != 5 then
return -1
endi
if $data50 != @21-03-24 19:26:38.231@ then
return -1
endi
if $data51 != 7 then
return -1
endi
if $data52 != 6 then
return -1
endi
if $data60 != @21-03-25 10:03:17.688@ then
return -1
endi
if $data61 != 8 then
return -1
endi
if $data62 != 7 then
return -1
endi
sql select add_one(f1),add_one(f1) from tb1;
if $rows != 7 then
return -1
endi
if $data00 != 2 then
return -1
endi
if $data01 != 2 then
return -1
endi
if $data10 != 3 then
return -1
endi
if $data11 != 3 then
return -1
endi
if $data20 != 4 then
return -1
endi
if $data21 != 4 then
return -1
endi
if $data30 != 5 then
return -1
endi
if $data31 != 5 then
return -1
endi
if $data40 != 6 then
return -1
endi
if $data41 != 6 then
return -1
endi
if $data50 != 7 then
return -1
endi
if $data51 != 7 then
return -1
endi
if $data60 != 8 then
return -1
endi
if $data61 != 8 then
return -1
endi
sql select add_one(f1)+1 from tb1;
if $rows != 7 then
return -1
endi
if $data00 != 3.000000000 then
return -1
endi
if $data10 != 4.000000000 then
return -1
endi
if $data20 != 5.000000000 then
return -1
endi
if $data30 != 6.000000000 then
return -1
endi
if $data40 != 7.000000000 then
return -1
endi
if $data50 != 8.000000000 then
return -1
endi
if $data60 != 9.000000000 then
return -1
endi
sql select sum_double(f1)+1 from tb1;
if $rows != 1 then
return -1
endi
if $data00 != 57.000000000 then
return -1
endi
sql select add_one(f1)+1,f1 from tb1;
if $rows != 7 then
return -1
endi
if $data00 != 3.000000000 then
return -1
endi
if $data01 != 1 then
return -1
endi
if $data10 != 4.000000000 then
return -1
endi
if $data11 != 2 then
return -1
endi
if $data20 != 5.000000000 then
return -1
endi
if $data21 != 3 then
return -1
endi
if $data30 != 6.000000000 then
return -1
endi
if $data31 != 4 then
return -1
endi
if $data40 != 7.000000000 then
return -1
endi
if $data41 != 5 then
return -1
endi
if $data50 != 8.000000000 then
return -1
endi
if $data51 != 6 then
return -1
endi
if $data60 != 9.000000000 then
return -1
endi
if $data61 != 7 then
return -1
endi
sql select sum_double(f1) from tb1 interval (10a);
if $rows != 7 then
return -1
endi
if $data00 != @21-03-23 17:17:19.660@ then
return -1
endi
if $data01 != 2 then
return -1
endi
if $data10 != @21-03-23 19:23:28.590@ then
return -1
endi
if $data11 != 4 then
return -1
endi
if $data20 != @21-03-23 19:33:39.070@ then
return -1
endi
if $data21 != 6 then
return -1
endi
if $data30 != @21-03-23 19:34:37.670@ then
return -1
endi
if $data31 != 8 then
return -1
endi
if $data40 != @21-03-24 19:08:06.600@ then
return -1
endi
if $data41 != 10 then
return -1
endi
if $data50 != @21-03-24 19:26:38.230@ then
return -1
endi
if $data51 != 12 then
return -1
endi
if $data60 != @21-03-25 10:03:17.680@ then
return -1
endi
if $data61 != 14 then
return -1
endi
sql select ts,add_one(f1) from tb1 where ts>="2021-03-23 17:00:00.000" and ts<="2021-03-24 20:00:00.000";
if $rows != 6 then
return -1
endi
if $data00 != @21-03-23 17:17:19.660@ then
return -1
endi
if $data01 != 2 then
return -1
endi
if $data10 != @21-03-23 19:23:28.595@ then
return -1
endi
if $data11 != 3 then
return -1
endi
if $data20 != @21-03-23 19:33:39.070@ then
return -1
endi
if $data21 != 4 then
return -1
endi
if $data30 != @21-03-23 19:34:37.670@ then
return -1
endi
if $data31 != 5 then
return -1
endi
if $data40 != @21-03-24 19:08:06.609@ then
return -1
endi
if $data41 != 6 then
return -1
endi
if $data50 != @21-03-24 19:26:38.231@ then
return -1
endi
if $data51 != 7 then
return -1
endi
sql select sum_double(f1) from tb1 where ts>="2021-03-23 17:00:00.000" and ts<="2021-03-24 20:00:00.000" interval (1h);
if $rows != 3 then
return -1
endi
if $data00 != @21-03-23 17:00:00.000@ then
return -1
endi
if $data01 != 2 then
return -1
endi
if $data10 != @21-03-23 19:00:00.000@ then
return -1
endi
if $data11 != 18 then
return -1
endi
if $data20 != @21-03-24 19:00:00.000@ then
return -1
endi
if $data21 != 22 then
return -1
endi
sql select sum_double(f1) from tb1 where ts>="2021-03-23 17:00:00.000" and ts<="2021-03-24 20:00:00.000" interval (1h) fill(value,999);
if $rows != 28 then
return -1
endi
sql_error select add_one(f1) from tb1 group by f1;
sql select sum_double(f1) from tb1 group by f1;
if $rows != 7 then
return -1
endi
if $data00 != 2 then
return -1
endi
if $data10 != 4 then
return -1
endi
if $data20 != 6 then
return -1
endi
if $data30 != 8 then
return -1
endi
if $data40 != 10 then
return -1
endi
if $data50 != 12 then
return -1
endi
if $data60 != 14 then
return -1
endi
sql select sum_double(f1) from tb1 interval (1h) order by ts desc;
if $rows != 4 then
return -1
endi
if $data00 != @21-03-25 10:00:00.000@ then
return -1
endi
if $data01 != 14 then
return -1
endi
if $data10 != @21-03-24 19:00:00.000@ then
return -1
endi
if $data11 != 22 then
return -1
endi
if $data20 != @21-03-23 19:00:00.000@ then
return -1
endi
if $data21 != 18 then
return -1
endi
if $data30 != @21-03-23 17:00:00.000@ then
return -1
endi
if $data31 != 2 then
return -1
endi
sql select add_one(f1) from tb1 limit 2;
if $rows != 2 then
return -1
endi
if $data00 != 2 then
return -1
endi
if $data10 != 3 then
return -1
endi
sql select sum_double(f1) from tb1 interval (1d) limit 2;
if $rows != 2 then
return -1
endi
if $data00 != @21-03-23 00:00:00.000@ then
return -1
endi
if $data01 != 20 then
return -1
endi
if $data10 != @21-03-24 00:00:00.000@ then
return -1
endi
if $data11 != 22 then
return -1
endi
sql_error select ts,sum_double(f1),f1 from tb1;
sql_error select add_one(f1),count(f1) from tb1;
sql_error select sum_double(f1),count(f1) from tb1;
sql_error select add_one(f1),top(f1,3) from tb1;
sql_error select add_one(f1) from tb1 interval(10a);
system sh/exec.sh -n dnode1 -s stop -x SIGINT

1163
tests/script/general/parser/udf_dll_stable.sim Normal file
View File

File diff suppressed because it is too large Load Diff

88
tests/script/sh/abs_max.c Normal file
View File

@ -0,0 +1,88 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
typedef struct SUdfInit{
int maybe_null; /* 1 if function can return NULL */
int decimals; /* for real functions */
long long length; /* For string functions */
char *ptr; /* free pointer for function data */
int const_item; /* 0 if result is independent of arguments */
} SUdfInit;
#define TSDB_DATA_INT_NULL 0x80000000L
#define TSDB_DATA_BIGINT_NULL 0x8000000000000000L
void abs_max(char* data, short itype, short ibytes, int numOfRows, long long* ts, char* dataOutput, char* interBuf, char* tsOutput,
int* numOfOutput, short otype, short obytes, SUdfInit* buf) {
int i;
int r = 0;
printf("abs_max input data:%p, type:%d, rows:%d, ts:%p,%lld, dataoutput:%p, tsOutput:%p, numOfOutput:%p, buf:%p\n", data, itype, numOfRows, ts, *ts, dataOutput, tsOutput, numOfOutput, buf);
if (itype == 5) {
r=*(long *)dataOutput;
*numOfOutput=0;
for(i=0;i<numOfRows;++i) {
if (*((long *)data + i) == TSDB_DATA_BIGINT_NULL) {
continue;
}
*numOfOutput=1;
long v = abs(*((long *)data + i));
if (v > r) {
r = v;
}
}
*(long *)dataOutput=r;
printf("abs_max out, dataoutput:%ld, numOfOutput:%d\n", *(long *)dataOutput, *numOfOutput);
}
}
void abs_max_finalize(char* dataOutput, char* interBuf, int* numOfOutput, SUdfInit* buf) {
int i;
int r = 0;
printf("abs_max_finalize dataoutput:%p:%d, numOfOutput:%d, buf:%p\n", dataOutput, *dataOutput, *numOfOutput, buf);
*numOfOutput=1;
printf("abs_max finalize, dataoutput:%ld, numOfOutput:%d\n", *(long *)dataOutput, *numOfOutput);
}
void abs_max_merge(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf) {
int r = 0;
if (numOfRows > 0) {
r = *((long *)data);
}
printf("abs_max_merge numOfRows:%d, dataoutput:%p, buf:%p\n", numOfRows, dataOutput, buf);
for (int i = 1; i < numOfRows; ++i) {
printf("abs_max_merge %d - %ld\n", i, *((long *)data + i));
if (*((long*)data + i) > r) {
r= *((long*)data + i);
}
}
*(long*)dataOutput=r;
if (numOfRows > 0) {
*numOfOutput=1;
} else {
*numOfOutput=0;
}
printf("abs_max_merge, dataoutput:%ld, numOfOutput:%d\n", *(long *)dataOutput, *numOfOutput);
}
int abs_max_init(SUdfInit* buf) {
printf("abs_max init\n");
return 0;
}
void abs_max_destroy(SUdfInit* buf) {
printf("abs_max destroy\n");
}

33
tests/script/sh/add_one.c Normal file
View File

@ -0,0 +1,33 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
typedef struct SUdfInit{
int maybe_null; /* 1 if function can return NULL */
int decimals; /* for real functions */
long long length; /* For string functions */
char *ptr; /* free pointer for function data */
int const_item; /* 0 if result is independent of arguments */
} SUdfInit;
void add_one(char* data, short itype, short ibytes, int numOfRows, long long* ts, char* dataOutput, char* interBUf, char* tsOutput,
int* numOfOutput, short otype, short obytes, SUdfInit* buf) {
int i;
int r = 0;
printf("add_one input data:%p, type:%d, rows:%d, ts:%p,%lld, dataoutput:%p, tsOutput:%p, numOfOutput:%p, buf:%p\n", data, itype, numOfRows, ts, *ts, dataOutput, tsOutput, numOfOutput, buf);
if (itype == 4) {
for(i=0;i<numOfRows;++i) {
printf("input %d - %d", i, *((int *)data + i));
*((int *)dataOutput+i)=*((int *)data + i) + 1;
printf(", output %d\n", *((int *)dataOutput+i));
if (tsOutput) {
*(long long*)tsOutput=1000000;
}
}
*numOfOutput=numOfRows;
printf("add_one out, numOfOutput:%d\n", *numOfOutput);
}
}

112
tests/script/sh/demo.c Normal file
View File

@ -0,0 +1,112 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
typedef struct SUdfInit{
int maybe_null; /* 1 if function can return NULL */
int decimals; /* for real functions */
long long length; /* For string functions */
char *ptr; /* free pointer for function data */
int const_item; /* 0 if result is independent of arguments */
} SUdfInit;
typedef struct SDemo{
double sum;
int num;
short otype;
}SDemo;
#define FLOAT_NULL 0x7FF00000 // it is an NAN
#define DOUBLE_NULL 0x7FFFFF0000000000L // it is an NAN
void demo(char* data, short itype, short ibytes, int numOfRows, long long* ts, char* dataOutput, char* interBuf, char* tsOutput,
int* numOfOutput, short otype, short obytes, SUdfInit* buf) {
int i;
double r = 0;
SDemo *p = (SDemo *)interBuf;
SDemo *q = (SDemo *)dataOutput;
printf("demo input data:%p, type:%d, rows:%d, ts:%p,%lld, dataoutput:%p, interBUf:%p, tsOutput:%p, numOfOutput:%p, buf:%p\n", data, itype, numOfRows, ts, *ts, dataOutput, interBuf, tsOutput, numOfOutput, buf);
for(i=0;i<numOfRows;++i) {
if (itype == 4) {
r=*((int *)data+i);
} else if (itype == 6) {
r=*((float *)data+i);
} else if (itype == 7) {
r=*((double *)data+i);
}
p->sum += r*r;
}
p->otype = otype;
p->num += numOfRows;
q->sum = p->sum;
q->num = p->num;
q->otype = p->otype;
*numOfOutput=1;
printf("demo out, sum:%f, num:%d, numOfOutput:%d\n", p->sum, p->num, *numOfOutput);
}
void demo_merge(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf) {
int i;
SDemo *p = (SDemo *)data;
SDemo res = {0};
printf("demo_merge input data:%p, rows:%d, dataoutput:%p, numOfOutput:%p, buf:%p\n", data, numOfRows, dataOutput, numOfOutput, buf);
for(i=0;i<numOfRows;++i) {
res.sum += p->sum * p->sum;
res.num += p->num;
p++;
}
p->sum = res.sum;
p->num = res.num;
*numOfOutput=1;
printf("demo out, sum:%f, num:%d, numOfOutput:%d\n", p->sum, p->num, *numOfOutput);
}
void demo_finalize(char* dataOutput, char* interBuf, int* numOfOutput, SUdfInit* buf) {
SDemo *p = (SDemo *)interBuf;
printf("demo_finalize interbuf:%p, numOfOutput:%p, buf:%p, sum:%f, num:%d\n", interBuf, numOfOutput, buf, p->sum, p->num);
if (p->otype == 6) {
if (p->num != 30000) {
*(unsigned int *)dataOutput = FLOAT_NULL;
} else {
*(float *)dataOutput = (float)(p->sum / p->num);
}
printf("finalize values:%f\n", *(float *)dataOutput);
} else if (p->otype == 7) {
if (p->num != 30000) {
*(unsigned long long *)dataOutput = DOUBLE_NULL;
} else {
*(double *)dataOutput = (double)(p->sum / p->num);
}
printf("finalize values:%f\n", *(double *)dataOutput);
}
*numOfOutput=1;
printf("demo finalize, numOfOutput:%d\n", *numOfOutput);
}
int demo_init(SUdfInit* buf) {
printf("demo init\n");
return 0;
}
void demo_destroy(SUdfInit* buf) {
printf("demo destroy\n");
}

43
tests/script/sh/demo.lua Normal file
View File

@ -0,0 +1,43 @@
funcName = "test"
global = {}
function test_init()
return global
end
function test_add(rows, ans, key)
t = {}
t["sum"] = 0.0
t["num"] = 0
for i=1, #rows do
t["sum"] = t["sum"] + rows[i] * rows[i]
end
t["num"] = #rows
if (ans[key] ~= nil)
then
ans[key]["sum"] = ans[key]["sum"] + t["sum"]
ans[key]["num"] = ans[key]["num"] + t["num"]
else
ans[key] = t
end
return ans;
end
function test_finalize(ans, key)
local ret = 0.0
if (ans[key] ~= nil and ans[key]["num"] == 30000)
then
ret = ans[key]["sum"]/ans[key]["num"]
ans[key]["sum"] = 0.0
ans[key]["num"] = 0
else
ret = inf
end
return ret, ans
end

12
tests/script/sh/prepare_udf.sh Normal file
View File

@ -0,0 +1,12 @@
#!/bin/sh
echo -n "hello" > /tmp/normal
echo -n "" > /tmp/empty
dd if=/dev/zero bs=3584 of=/tmp/big count=1
rm -rf /tmp/sum_double.so /tmp/add_one.so
gcc -g -O0 -fPIC -shared sh/sum_double.c -o /tmp/sum_double.so
gcc -g -O0 -fPIC -shared sh/add_one.c -o /tmp/add_one.so
gcc -g -O0 -fPIC -shared sh/demo.c -o /tmp/demo.so
gcc -g -O0 -fPIC -shared sh/abs_max.c -o /tmp/abs_max.so

14
tests/script/sh/scalar_demo.lua Normal file
View File

@ -0,0 +1,14 @@
funcName = "test"
ans = 1
function test_init()
return ans
end
function test_add(rows, ans)
for i=1, #rows do
rows[i] = rows[i] + ans
end
return rows
end

84
tests/script/sh/sum_double.c Normal file
View File

@ -0,0 +1,84 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
typedef struct SUdfInit{
int maybe_null; /* 1 if function can return NULL */
int decimals; /* for real functions */
long long length; /* For string functions */
char *ptr; /* free pointer for function data */
int const_item; /* 0 if result is independent of arguments */
} SUdfInit;
#define TSDB_DATA_INT_NULL 0x80000000L
void sum_double(char* data, short itype, short ibytes, int numOfRows, long long* ts, char* dataOutput, char* interBuf, char* tsOutput,
int* numOfOutput, short otype, short obytes, SUdfInit* buf) {
int i;
int r = 0;
printf("sum_double input data:%p, type:%d, rows:%d, ts:%p,%lld, dataoutput:%p, tsOutput:%p, numOfOutput:%p, buf:%p\n", data, itype, numOfRows, ts, *ts, dataOutput, tsOutput, numOfOutput, buf);
if (itype == 4) {
r=*(int *)dataOutput;
*numOfOutput=0;
for(i=0;i<numOfRows;++i) {
if (*((int *)data + i) == TSDB_DATA_INT_NULL) {
continue;
}
*numOfOutput=1;
r+=*((int *)data + i);
*(int *)dataOutput=r;
}
printf("sum_double out, dataoutput:%d, numOfOutput:%d\n", *(int *)dataOutput, *numOfOutput);
}
}
void sum_double_finalize(char* dataOutput, char* interBuf, int* numOfOutput, SUdfInit* buf) {
int i;
int r = 0;
printf("sum_double_finalize dataoutput:%p:%d, numOfOutput:%d, buf:%p\n", dataOutput, *dataOutput, *numOfOutput, buf);
*numOfOutput=1;
*(int*)(buf->ptr)=*(int*)dataOutput*2;
*(int*)dataOutput=*(int*)(buf->ptr);
printf("sum_double finalize, dataoutput:%d, numOfOutput:%d\n", *(int *)dataOutput, *numOfOutput);
}
void sum_double_merge(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf) {
int r = 0;
int sum = 0;
printf("sum_double_merge numOfRows:%d, dataoutput:%p, buf:%p\n", numOfRows, dataOutput, buf);
for (int i = 0; i < numOfRows; ++i) {
printf("sum_double_merge %d - %d\n", i, *((int*)data + i));
sum +=*((int*)data + i);
}
*(int*)dataOutput+=sum;
if (numOfRows > 0) {
*numOfOutput=1;
} else {
*numOfOutput=0;
}
printf("sum_double_merge, dataoutput:%d, numOfOutput:%d\n", *(int *)dataOutput, *numOfOutput);
}
int sum_double_init(SUdfInit* buf) {
buf->maybe_null=1;
buf->ptr = malloc(sizeof(int));
printf("sum_double init\n");
return 0;
}
void sum_double_destroy(SUdfInit* buf) {
free(buf->ptr);
printf("sum_double destroy\n");
}