diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 83283cfdad..1f7dbb8b62 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -125,6 +125,10 @@ extern SDiskCfg tsDiskCfg[]; // udf extern bool tsStartUdfd; +// schemaless +extern char tsSmlChildTableName[]; +extern bool tsSmlDataFormat; + // internal extern int32_t tsTransPullupInterval; extern int32_t tsMqRebalanceInterval; diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 2b58ed7c0b..aec1476663 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -41,6 +41,7 @@ typedef enum EFunctionType { FUNCTION_TYPE_SUM, FUNCTION_TYPE_TWA, FUNCTION_TYPE_HISTOGRAM, + FUNCTION_TYPE_HYPERLOGLOG, // nonstandard SQL function FUNCTION_TYPE_BOTTOM = 500, diff --git a/include/libs/function/tudf.h b/include/libs/function/tudf.h index 6a98138c6c..28b1fbe8ce 100644 --- a/include/libs/function/tudf.h +++ b/include/libs/function/tudf.h @@ -87,6 +87,7 @@ typedef struct SUdfInterBuf { } SUdfInterBuf; typedef void *UdfcFuncHandle; +//low level APIs /** * setup udf * @param udf, in @@ -115,6 +116,9 @@ int32_t doCallUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t */ int32_t doTeardownUdf(UdfcFuncHandle handle); +void freeUdfInterBuf(SUdfInterBuf *buf); + +//high level APIs bool udfAggGetEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); int32_t udfAggProcess(struct SqlFunctionCtx *pCtx); diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 60f00daebe..01f3132c69 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -325,6 +325,7 @@ typedef struct SQuery { bool showRewrite; int32_t placeholderNum; SArray* pPlaceholderValues; + SNode* pContainPlaceholderRoot; } SQuery; void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext); diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 248953cbaf..2d8fd9a93c 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -62,7 +62,7 @@ int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc, uint64_t uid, int32_t void qDestroyStmtDataBlock(void* pBlock); STableMeta* qGetTableMetaInDataBlock(void* pDataBlock); -int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx, uint64_t queryId); +int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx); int32_t qStmtParseQuerySql(SParseContext* pCxt, SQuery* pQuery); int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen); int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx, @@ -77,8 +77,8 @@ int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* void* smlInitHandle(SQuery* pQuery); void smlDestroyHandle(void* pHandle); -int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols, bool format, - STableMeta* pTableMeta, char* tableName, char* msgBuf, int16_t msgBufLen); +int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols, bool format, STableMeta* pTableMeta, + char* tableName, char* msgBuf, int16_t msgBufLen); int32_t smlBuildOutput(void* handle, SHashObj* pVgHash); #ifdef __cplusplus diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index e250b7b2b2..c4f71e57a8 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -34,7 +34,6 @@ typedef struct SPlanContext { bool showRewrite; int8_t triggerType; int64_t watermark; - int32_t placeholderNum; char* pMsg; int32_t msgLen; } SPlanContext; @@ -48,9 +47,6 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo // @pSource one execution location of this group of datasource subplans int32_t qSetSubplanExecutionNode(SSubplan* pSubplan, int32_t groupId, SDownstreamSourceNode* pSource); -int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_MULTI_BIND* pParams, int32_t colIdx, uint64_t queryId, - bool* pEmptyResult); - // Convert to subplan to string for the scheduler to send to the executor int32_t qSubPlanToString(const SSubplan* pSubplan, char** pStr, int32_t* pLen); int32_t qStringToSubplan(const char* pStr, SSubplan** pSubplan); diff --git a/include/os/os.h b/include/os/os.h index 329ad481aa..41180ba49e 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -58,9 +58,6 @@ extern "C" { #else #include #endif - -#define __typeof(a) auto - #endif #include diff --git a/include/os/osFile.h b/include/os/osFile.h index 5ba161270d..8751e175a5 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -66,7 +66,7 @@ int32_t taosUnLockFile(TdFilePtr pFile); int32_t taosUmaskFile(int32_t maskVal); int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime); -int32_t taosDevInoFile(const char *path, int64_t *stDev, int64_t *stIno); +int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno); int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime); bool taosCheckExistFile(const char *pathname); diff --git a/include/os/osMath.h b/include/os/osMath.h index 829bbd847b..f17ca56c9e 100644 --- a/include/os/osMath.h +++ b/include/os/osMath.h @@ -23,11 +23,13 @@ extern "C" { #define TPOW2(x) ((x) * (x)) #define TABS(x) ((x) > 0 ? (x) : -(x)) -#define TSWAP(a, b) \ - do { \ - __typeof(a) __tmp = (a); \ - (a) = (b); \ - (b) = __tmp; \ +#define TSWAP(a, b) \ + do { \ + char *__tmp = taosMemoryMalloc(sizeof(a)); \ + memcpy(__tmp, &(a), sizeof(a)); \ + memcpy(&(a), &(b), sizeof(a)); \ + memcpy(&(b), __tmp, sizeof(a)); \ + taosMemoryFree(__tmp); \ } while (0) #ifdef WINDOWS diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 31b6dad2d8..f2ab9cbc6e 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -644,6 +644,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_INVALID_TIMELINE_FUNC TAOS_DEF_ERROR_CODE(0, 0x2647) #define TSDB_CODE_PAR_INVALID_PASSWD TAOS_DEF_ERROR_CODE(0, 0x2648) #define TSDB_CODE_PAR_INVALID_ALTER_TABLE TAOS_DEF_ERROR_CODE(0, 0x2649) +#define TSDB_CODE_PAR_CANNOT_DROP_PRIMARY_KEY TAOS_DEF_ERROR_CODE(0, 0x264A) +#define TSDB_CODE_PAR_INVALID_MODIFY_COL TAOS_DEF_ERROR_CODE(0, 0x264B) //planner #define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700) diff --git a/include/util/tcoding.h b/include/util/tcoding.h index 74e64d5292..5962949a70 100644 --- a/include/util/tcoding.h +++ b/include/util/tcoding.h @@ -63,14 +63,14 @@ static FORCE_INLINE void *taosSkipFixedLen(const void *buf, size_t len) { return static FORCE_INLINE int32_t taosEncodeFixedBool(void **buf, bool value) { if (buf != NULL) { - ((int8_t *)(*buf))[0] = value ? 1 : 0; + ((int8_t *)(*buf))[0] = (value ? 1 : 0); *buf = POINTER_SHIFT(*buf, sizeof(int8_t)); } return (int32_t)sizeof(int8_t); } static FORCE_INLINE void *taosDecodeFixedBool(const void *buf, bool *value) { - *value = ((int8_t *)buf)[0] == 0 ? false : true; + *value = ( (((int8_t *)buf)[0] == 0) ? false : true ); return POINTER_SHIFT(buf, sizeof(int8_t)); } diff --git a/include/util/thash.h b/include/util/thash.h index f2ef445777..fc8785a8fb 100644 --- a/include/util/thash.h +++ b/include/util/thash.h @@ -40,6 +40,7 @@ typedef void (*_hash_free_fn_t)(void *); */ uint32_t MurmurHash3_32(const char *key, uint32_t len); +uint64_t MurmurHash3_64(const char *key, uint32_t len); /** * * @param key diff --git a/include/util/tjson.h b/include/util/tjson.h index a95efe56e7..84f7b81726 100644 --- a/include/util/tjson.h +++ b/include/util/tjson.h @@ -25,7 +25,7 @@ extern "C" { #define tjsonGetNumberValue(pJson, pName, val, code) \ do { \ uint64_t _tmp = 0; \ - code = tjsonGetUBigIntValue(pJson, pName, &_tmp); \ + code = tjsonGetBigIntValue(pJson, pName, &_tmp); \ val = _tmp; \ } while (0) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 100ec42899..de08dbb34f 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -233,8 +233,7 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra .pAstRoot = pQuery->pRoot, .showRewrite = pQuery->showRewrite, .pMsg = pRequest->msgBuf, - .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, - .placeholderNum = pQuery->placeholderNum}; + .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE}; SEpSet mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); SCatalog* pCatalog = NULL; int32_t code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); @@ -518,7 +517,7 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t if (pRequest->code != TSDB_CODE_SUCCESS) { const char* errorMsg = (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) ? taos_errstr(pRequest) : tstrerror(pRequest->code); - fprintf(stderr,"failed to connect to server, reason: %s\n\n", errorMsg); + fprintf(stderr, "failed to connect to server, reason: %s\n\n", errorMsg); terrno = pRequest->code; destroyRequest(pRequest); @@ -949,7 +948,8 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR // TODO handle the compressed case pResultInfo->totalRows += pResultInfo->numOfRows; - return setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows, convertUcs4); + return setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows, + convertUcs4); } TSDB_SERVER_STATUS taos_check_server_status(const char* fqdn, int port, char* details, int maxlen) { diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 8fce5bfb00..884756ced9 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -16,6 +16,7 @@ #include "clientInt.h" #include "tname.h" #include "cJSON.h" +#include "tglobal.h" //================================================================================================= #define SPACE ' ' @@ -54,6 +55,9 @@ for (int i = 1; i < keyLen; ++i) { \ } \ } +#define IS_INVALID_COL_LEN(len) ((len) <= 0 || (len) >= TSDB_COL_NAME_LEN) +#define IS_INVALID_TABLE_LEN(len) ((len) <= 0 || (len) >= TSDB_TABLE_NAME_LEN) + #define OTD_MAX_FIELDS_NUM 2 #define OTD_JSON_SUB_FIELDS_NUM 2 #define OTD_JSON_FIELDS_NUM 4 @@ -162,7 +166,7 @@ typedef struct { SMLProtocolType protocol; int8_t precision; - bool dataFormat; // true means that the name, number and order of keys in each line are the same(only for influx protocol) + bool dataFormat; // true means that the name and order of keys in each line are the same(only for influx protocol) SHashObj *childTables; SHashObj *superTables; @@ -594,19 +598,25 @@ static bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg){ kvVal->type = TSDB_DATA_TYPE_FLOAT; kvVal->f = (float)result; }else if ((left == 1 && *endptr == 'i') || (left == 3 && strncasecmp(endptr, "i64", left) == 0)){ - if(smlDoubleToInt64OverFlow(result)){ - smlBuildInvalidDataMsg(msg, "big int is too large, out of precision", pVal); - return false; + if(result >= (double)INT64_MAX){ + kvVal->i = INT64_MAX; + }else if(result <= (double)INT64_MIN){ + kvVal->i = INT64_MIN; + }else{ + kvVal->i = result; } kvVal->type = TSDB_DATA_TYPE_BIGINT; - kvVal->i = (int64_t)result; }else if ((left == 3 && strncasecmp(endptr, "u64", left) == 0)){ - if(result >= (double)UINT64_MAX || result < 0){ + if(result < 0){ smlBuildInvalidDataMsg(msg, "unsigned big int is too large, out of precision", pVal); return false; } + if(result >= (double)UINT64_MAX){ + kvVal->u = UINT64_MAX; + }else{ + kvVal->u = result; + } kvVal->type = TSDB_DATA_TYPE_UBIGINT; - kvVal->u = result; }else if (left == 3 && strncasecmp(endptr, "i32", left) == 0){ if(!IS_VALID_INT(result)){ smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal); @@ -659,12 +669,12 @@ static bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg){ static bool smlParseBool(SSmlKv *kvVal) { const char *pVal = kvVal->value; int32_t len = kvVal->length; - if ((len == 1) && pVal[0] == 't') { + if ((len == 1) && (pVal[0] == 't' || pVal[0] == 'T')) { kvVal->i = true; return true; } - if ((len == 1) && pVal[0] == 'f') { + if ((len == 1) && (pVal[0] == 'f' || pVal[0] == 'F')) { kvVal->i = false; return true; } @@ -899,8 +909,8 @@ static int32_t smlParseInfluxString(const char* sql, SSmlLineInfo *elements, SSm sql++; } elements->measureLen = sql - elements->measure; - if(elements->measureLen == 0) { - smlBuildInvalidDataMsg(msg, "measure is empty", NULL); + if(IS_INVALID_TABLE_LEN(elements->measureLen)) { + smlBuildInvalidDataMsg(msg, "measure is empty or too large than 192", NULL); return TSDB_CODE_SML_INVALID_DATA; } @@ -969,8 +979,9 @@ static void smlParseTelnetElement(const char **sql, const char **data, int32_t * } } -static int32_t smlParseTelnetTags(const char* data, SArray *cols, SHashObj *dumplicateKey, SSmlMsgBuf *msg){ +static int32_t smlParseTelnetTags(const char* data, SArray *cols, char *childTableName, SHashObj *dumplicateKey, SSmlMsgBuf *msg){ const char *sql = data; + size_t childTableNameLen = strlen(tsSmlChildTableName); while(*sql != '\0'){ JUMP_SPACE(sql) if(*sql == '\0') break; @@ -992,7 +1003,7 @@ static int32_t smlParseTelnetTags(const char* data, SArray *cols, SHashObj *dump sql++; } - if(keyLen == 0 || keyLen >= TSDB_COL_NAME_LEN){ + if(IS_INVALID_COL_LEN(keyLen)){ smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key); return TSDB_CODE_SML_INVALID_DATA; } @@ -1022,6 +1033,13 @@ static int32_t smlParseTelnetTags(const char* data, SArray *cols, SHashObj *dump return TSDB_CODE_SML_INVALID_DATA; } + //handle child table name + if(childTableNameLen != 0 && strncmp(key, tsSmlChildTableName, keyLen) == 0){ + memset(childTableName, 0, TSDB_TABLE_NAME_LEN); + strncpy(childTableName, value, (valueLen < TSDB_TABLE_NAME_LEN ? valueLen : TSDB_TABLE_NAME_LEN)); + continue; + } + // add kv to SSmlKv SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); if(!kv) return TSDB_CODE_OUT_OF_MEMORY; @@ -1043,7 +1061,7 @@ static int32_t smlParseTelnetString(SSmlHandle *info, const char* sql, SSmlTable // parse metric smlParseTelnetElement(&sql, &tinfo->sTableName, &tinfo->sTableNameLen); - if (!(tinfo->sTableName) || tinfo->sTableNameLen == 0) { + if (!(tinfo->sTableName) || IS_INVALID_TABLE_LEN(tinfo->sTableNameLen)) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); return TSDB_CODE_SML_INVALID_DATA; } @@ -1085,7 +1103,7 @@ static int32_t smlParseTelnetString(SSmlHandle *info, const char* sql, SSmlTable } // parse tags - ret = smlParseTelnetTags(sql, tinfo->tags, info->dumplicateKey, &info->msgBuf); + ret = smlParseTelnetTags(sql, tinfo->tags, tinfo->childTableName, info->dumplicateKey, &info->msgBuf); if (ret != TSDB_CODE_SUCCESS) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); return TSDB_CODE_SML_INVALID_DATA; @@ -1094,7 +1112,7 @@ static int32_t smlParseTelnetString(SSmlHandle *info, const char* sql, SSmlTable return TSDB_CODE_SUCCESS; } -static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, bool isTag, SHashObj *dumplicateKey, SSmlMsgBuf *msg){ +static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, char *childTableName, bool isTag, SHashObj *dumplicateKey, SSmlMsgBuf *msg){ if(isTag && len == 0){ SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); if(!kv) return TSDB_CODE_OUT_OF_MEMORY; @@ -1107,6 +1125,7 @@ static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, bool is return TSDB_CODE_SUCCESS; } + size_t childTableNameLen = strlen(tsSmlChildTableName); const char *sql = data; while(sql < data + len){ const char *key = sql; @@ -1126,7 +1145,7 @@ static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, bool is sql++; } - if(keyLen == 0 || keyLen >= TSDB_COL_NAME_LEN){ + if(IS_INVALID_COL_LEN(keyLen)){ smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key); return TSDB_CODE_SML_INVALID_DATA; } @@ -1169,6 +1188,13 @@ static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, bool is PROCESS_SLASH(key, keyLen) PROCESS_SLASH(value, valueLen) + //handle child table name + if(childTableName && childTableNameLen != 0 && strncmp(key, tsSmlChildTableName, keyLen) == 0){ + memset(childTableName, 0, TSDB_TABLE_NAME_LEN); + strncpy(childTableName, value, (valueLen < TSDB_TABLE_NAME_LEN ? valueLen : TSDB_TABLE_NAME_LEN)); + continue; + } + // add kv to SSmlKv SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); if(!kv) return TSDB_CODE_OUT_OF_MEMORY; @@ -1396,7 +1422,7 @@ static void smlDestroyInfo(SSmlHandle* info){ taosMemoryFreeClear(info); } -static SSmlHandle* smlBuildSmlInfo(TAOS* taos, SRequestObj* request, SMLProtocolType protocol, int8_t precision, bool dataFormat){ +static SSmlHandle* smlBuildSmlInfo(TAOS* taos, SRequestObj* request, SMLProtocolType protocol, int8_t precision){ int32_t code = TSDB_CODE_SUCCESS; SSmlHandle* info = (SSmlHandle*)taosMemoryCalloc(1, sizeof(SSmlHandle)); if (NULL == info) { @@ -1428,7 +1454,11 @@ static SSmlHandle* smlBuildSmlInfo(TAOS* taos, SRequestObj* request, SMLProtocol info->precision = precision; info->protocol = protocol; - info->dataFormat = dataFormat; + if(protocol == TSDB_SML_LINE_PROTOCOL){ + info->dataFormat = tsSmlDataFormat; + }else{ + info->dataFormat = true; + } info->pRequest = request; info->msgBuf.buf = info->pRequest->msgBuf; info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE; @@ -1439,7 +1469,7 @@ static SSmlHandle* smlBuildSmlInfo(TAOS* taos, SRequestObj* request, SMLProtocol info->pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); info->dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if(!dataFormat){ + if(!info->dataFormat){ info->colsContainer = taosArrayInit(32, POINTER_BYTES); if(NULL == info->colsContainer){ uError("SML:0x%"PRIx64" create info failed", info->id); @@ -1477,8 +1507,8 @@ static int32_t smlParseMetricFromJSON(SSmlHandle *info, cJSON *root, SSmlTableIn } tinfo->sTableNameLen = strlen(metric->valuestring); - if (tinfo->sTableNameLen >= TSDB_TABLE_NAME_LEN) { - uError("OTD:0x%"PRIx64" Metric cannot exceeds %d characters in JSON", info->id, TSDB_TABLE_NAME_LEN - 1); + if (IS_INVALID_TABLE_LEN(tinfo->sTableNameLen)) { + uError("OTD:0x%"PRIx64" Metric lenght is 0 or large than 192", info->id); return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; } @@ -1674,11 +1704,13 @@ static int32_t smlConvertJSONNumber(SSmlKv *pVal, char* typeStr, cJSON *value) { strcasecmp(typeStr, "bigint") == 0) { pVal->type = TSDB_DATA_TYPE_BIGINT; pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - if(smlDoubleToInt64OverFlow(value->valuedouble)){ - uError("OTD:JSON value(%f) cannot fit in type(big int)", value->valuedouble); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + if(value->valuedouble >= (double)INT64_MAX){ + pVal->i = INT64_MAX; + }else if(value->valuedouble <= (double)INT64_MIN){ + pVal->i = INT64_MIN; + }else{ + pVal->i = value->valuedouble; } - pVal->i = value->valuedouble; return TSDB_CODE_SUCCESS; } //float @@ -1828,60 +1860,49 @@ static int32_t smlParseColsFromJSON(cJSON *root, SArray *cols) { return TSDB_CODE_SUCCESS; } -static int32_t smlParseTagsFromJSON(cJSON *root, SArray *pKVs, SHashObj *dumplicateKey, SSmlMsgBuf *msg) { +static int32_t smlParseTagsFromJSON(cJSON *root, SArray *pKVs, char *childTableName, SHashObj *dumplicateKey, SSmlMsgBuf *msg) { int32_t ret = TSDB_CODE_SUCCESS; cJSON *tags = cJSON_GetObjectItem(root, "tags"); if (tags == NULL || tags->type != cJSON_Object) { return TSDB_CODE_TSC_INVALID_JSON; } - //handle child table name todo -// size_t childTableNameLen = strlen(tsSmlChildTableName); -// char childTbName[TSDB_TABLE_NAME_LEN] = {0}; -// if (childTableNameLen != 0) { -// memcpy(childTbName, tsSmlChildTableName, childTableNameLen); -// cJSON *id = cJSON_GetObjectItem(tags, childTbName); -// if (id != NULL) { -// if (!cJSON_IsString(id)) { -// tscError("OTD:0x%"PRIx64" ID must be JSON string", info->id); -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// size_t idLen = strlen(id->valuestring); -// *childTableName = tcalloc(idLen + TS_BACKQUOTE_CHAR_SIZE + 1, sizeof(char)); -// memcpy(*childTableName, id->valuestring, idLen); -// addEscapeCharToString(*childTableName, (int32_t)idLen); -// -// //check duplicate IDs -// cJSON_DeleteItemFromObject(tags, childTbName); -// id = cJSON_GetObjectItem(tags, childTbName); -// if (id != NULL) { -// return TSDB_CODE_TSC_DUP_TAG_NAMES; -// } -// } -// } + size_t childTableNameLen = strlen(tsSmlChildTableName); int32_t tagNum = cJSON_GetArraySize(tags); for (int32_t i = 0; i < tagNum; ++i) { cJSON *tag = cJSON_GetArrayItem(tags, i); if (tag == NULL) { return TSDB_CODE_TSC_INVALID_JSON; } + size_t keyLen = strlen(tag->string); + if (IS_INVALID_COL_LEN(keyLen)) { + uError("OTD:Tag key length is 0 or too large than 64"); + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + } //check duplicate keys - if (smlCheckDuplicateKey(tag->string, strlen(tag->string), dumplicateKey)) { + if (smlCheckDuplicateKey(tag->string, keyLen, dumplicateKey)) { return TSDB_CODE_TSC_DUP_TAG_NAMES; } + //handle child table name + if(childTableNameLen != 0 && strcmp(tag->string, tsSmlChildTableName) == 0){ + if (!cJSON_IsString(tag)) { + uError("OTD:ID must be JSON string"); + return TSDB_CODE_TSC_INVALID_JSON; + } + memset(childTableName, 0, TSDB_TABLE_NAME_LEN); + strncpy(childTableName, tag->valuestring, TSDB_TABLE_NAME_LEN); + continue; + } + // add kv to SSmlKv SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); if(!kv) return TSDB_CODE_OUT_OF_MEMORY; if(pKVs) taosArrayPush(pKVs, &kv); //key - kv->keyLen = strlen(tag->string); - if (kv->keyLen >= TSDB_COL_NAME_LEN) { - uError("OTD:Tag key cannot exceeds %d characters in JSON", TSDB_COL_NAME_LEN - 1); - return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - } + kv->keyLen = keyLen; ret = smlJsonCreateSring(&kv->key, tag->string, kv->keyLen); if (ret != TSDB_CODE_SUCCESS) { return ret; @@ -1937,7 +1958,7 @@ static int32_t smlParseJSONString(SSmlHandle *info, cJSON *root, SSmlTableInfo * uDebug("OTD:0x%"PRIx64" Parse metric value from JSON payload finished", info->id); //Parse tags - ret = smlParseTagsFromJSON(root, tinfo->tags, info->dumplicateKey, &info->msgBuf); + ret = smlParseTagsFromJSON(root, tinfo->tags, tinfo->childTableName, info->dumplicateKey, &info->msgBuf); if (ret) { uError("OTD:0x%"PRIx64" Unable to parse tags from JSON payload", info->id); return ret; @@ -1975,7 +1996,7 @@ static int32_t smlParseInfluxLine(SSmlHandle* info, const char* sql) { if(info->dataFormat) taosArrayDestroy(cols); return ret; } - ret = smlParseCols(elements.cols, elements.colsLen, cols, false, info->dumplicateKey, &info->msgBuf); + ret = smlParseCols(elements.cols, elements.colsLen, cols, NULL, false, info->dumplicateKey, &info->msgBuf); if(ret != TSDB_CODE_SUCCESS){ uError("SML:0x%"PRIx64" smlParseCols parse cloums fields failed", info->id); smlDestroyCols(cols); @@ -2006,7 +2027,7 @@ static int32_t smlParseInfluxLine(SSmlHandle* info, const char* sql) { } if(!hasTable){ - ret = smlParseCols(elements.tags, elements.tagsLen, (*oneTable)->tags, true, info->dumplicateKey, &info->msgBuf); + ret = smlParseCols(elements.tags, elements.tagsLen, (*oneTable)->tags, (*oneTable)->childTableName, true, info->dumplicateKey, &info->msgBuf); if(ret != TSDB_CODE_SUCCESS){ uError("SML:0x%"PRIx64" smlParseCols parse tag fields failed", info->id); return ret; @@ -2019,11 +2040,16 @@ static int32_t smlParseInfluxLine(SSmlHandle* info, const char* sql) { (*oneTable)->sTableName = elements.measure; (*oneTable)->sTableNameLen = elements.measureLen; - RandTableName rName = { (*oneTable)->tags, (*oneTable)->sTableName, (uint8_t)(*oneTable)->sTableNameLen, - (*oneTable)->childTableName, 0 }; + if(strlen((*oneTable)->childTableName) == 0){ + RandTableName rName = { (*oneTable)->tags, (*oneTable)->sTableName, (uint8_t)(*oneTable)->sTableNameLen, + (*oneTable)->childTableName, 0 }; + + buildChildTableName(&rName); + (*oneTable)->uid = rName.uid; + }else{ + (*oneTable)->uid = *(uint64_t*)((*oneTable)->childTableName); + } - buildChildTableName(&rName); - (*oneTable)->uid = rName.uid; } SSmlSTableMeta** tableMeta = (SSmlSTableMeta**)taosHashGet(info->superTables, elements.measure, elements.measureLen); @@ -2087,10 +2113,15 @@ static int32_t smlParseTelnetLine(SSmlHandle* info, void *data) { } taosHashClear(info->dumplicateKey); - RandTableName rName = { tinfo->tags, tinfo->sTableName, (uint8_t)tinfo->sTableNameLen, - tinfo->childTableName, 0 }; - buildChildTableName(&rName); - tinfo->uid = rName.uid; + if(strlen(tinfo->childTableName) == 0){ + RandTableName rName = { tinfo->tags, tinfo->sTableName, (uint8_t)tinfo->sTableNameLen, + tinfo->childTableName, 0 }; + buildChildTableName(&rName); + tinfo->uid = rName.uid; + }else{ + tinfo->uid = *(uint64_t*)(tinfo->childTableName); // generate uid by name simple + } + bool hasTable = true; SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashGet(info->childTables, tinfo->childTableName, strlen(tinfo->childTableName)); @@ -2308,14 +2339,14 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr return NULL; } - SSmlHandle* info = smlBuildSmlInfo(taos, request, (SMLProtocolType)protocol, precision, true); + SSmlHandle* info = smlBuildSmlInfo(taos, request, (SMLProtocolType)protocol, precision); if(!info){ return (TAOS_RES*)request; } - if (numLines <= 0 || numLines > 65536) { + if (!lines) { request->code = TSDB_CODE_SML_INVALID_DATA; - smlBuildInvalidDataMsg(&info->msgBuf, "numLines should be between 1 and 65536", NULL); + smlBuildInvalidDataMsg(&info->msgBuf, "lines is null", NULL); goto end; } @@ -2325,7 +2356,7 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr goto end; } - if(protocol == TSDB_SML_LINE_PROTOCOL && (precision < TSDB_SML_TIMESTAMP_HOURS || precision > TSDB_SML_TIMESTAMP_NANO_SECONDS)){ + if(protocol == TSDB_SML_LINE_PROTOCOL && (precision < TSDB_SML_TIMESTAMP_NOT_CONFIGURED || precision > TSDB_SML_TIMESTAMP_NANO_SECONDS)){ request->code = TSDB_CODE_SML_INVALID_PRECISION_TYPE; smlBuildInvalidDataMsg(&info->msgBuf, "precision invalidate for line protocol", NULL); goto end; diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 1e9cb7b24f..fa8df28523 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -67,7 +67,7 @@ int32_t stmtGetTbName(TAOS_STMT* stmt, char** tbName) { STscStmt* pStmt = (STscStmt*)stmt; pStmt->sql.type = STMT_TYPE_MULTI_INSERT; - + if ('\0' == pStmt->bInfo.tbName[0]) { tscError("no table name set"); STMT_ERR_RET(TSDB_CODE_TSC_STMT_TBNAME_ERROR); @@ -126,7 +126,7 @@ int32_t stmtUpdateBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, strncpy(pStmt->bInfo.tbFName, tbFName, sizeof(pStmt->bInfo.tbFName) - 1); pStmt->bInfo.tbFName[sizeof(pStmt->bInfo.tbFName) - 1] = 0; - + pStmt->bInfo.tbUid = pTableMeta->uid; pStmt->bInfo.tbSuid = pTableMeta->suid; pStmt->bInfo.tbType = pTableMeta->tableType; @@ -146,18 +146,18 @@ int32_t stmtUpdateExecInfo(TAOS_STMT* stmt, SHashObj* pVgHash, SHashObj* pBlockH return TSDB_CODE_SUCCESS; } -int32_t stmtUpdateInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, char* tbFName, bool autoCreateTbl, SHashObj* pVgHash, SHashObj* pBlockHash) { +int32_t stmtUpdateInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, char* tbFName, bool autoCreateTbl, + SHashObj* pVgHash, SHashObj* pBlockHash) { STscStmt* pStmt = (STscStmt*)stmt; STMT_ERR_RET(stmtUpdateBindInfo(stmt, pTableMeta, tags, tbFName)); STMT_ERR_RET(stmtUpdateExecInfo(stmt, pVgHash, pBlockHash, autoCreateTbl)); pStmt->sql.autoCreateTbl = autoCreateTbl; - + return TSDB_CODE_SUCCESS; } - int32_t stmtGetExecInfo(TAOS_STMT* stmt, SHashObj** pVgHash, SHashObj** pBlockHash) { STscStmt* pStmt = (STscStmt*)stmt; @@ -172,7 +172,7 @@ int32_t stmtCacheBlock(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } - uint64_t uid = pStmt->bInfo.tbUid; + uint64_t uid = pStmt->bInfo.tbUid; uint64_t cacheUid = (TSDB_CHILD_TABLE == pStmt->bInfo.tbType) ? pStmt->bInfo.tbSuid : uid; if (taosHashGet(pStmt->sql.pTableCache, &cacheUid, sizeof(cacheUid))) { @@ -180,8 +180,8 @@ int32_t stmtCacheBlock(STscStmt* pStmt) { } STableDataBlocks** pSrc = taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); - STableDataBlocks* pDst = NULL; - + STableDataBlocks* pDst = NULL; + STMT_ERR_RET(qCloneStmtDataBlock(&pDst, *pSrc)); SStmtTableCache cache = { @@ -198,16 +198,16 @@ int32_t stmtCacheBlock(STscStmt* pStmt) { } else { pStmt->bInfo.boundTags = NULL; } - + return TSDB_CODE_SUCCESS; } int32_t stmtParseSql(STscStmt* pStmt) { SStmtCallback stmtCb = { - .pStmt = pStmt, - .getTbNameFn = stmtGetTbName, - .setInfoFn = stmtUpdateInfo, - .getExecInfoFn = stmtGetExecInfo, + .pStmt = pStmt, + .getTbNameFn = stmtGetTbName, + .setInfoFn = stmtUpdateInfo, + .getExecInfoFn = stmtGetExecInfo, }; if (NULL == pStmt->exec.pRequest) { @@ -259,12 +259,12 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool freeRequest) { } size_t keyLen = 0; - void *pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL); + void* pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL); while (pIter) { - STableDataBlocks* pBlocks = *(STableDataBlocks**)pIter; - char *key = taosHashGetKey(pIter, &keyLen); - STableMeta* pMeta = qGetTableMetaInDataBlock(pBlocks); - + STableDataBlocks* pBlocks = *(STableDataBlocks**)pIter; + char* key = taosHashGetKey(pIter, &keyLen); + STableMeta* pMeta = qGetTableMetaInDataBlock(pBlocks); + if (keepTable && (strlen(pStmt->bInfo.tbFName) == keyLen) && strncmp(pStmt->bInfo.tbFName, key, keyLen) == 0) { STMT_ERR_RET(qResetStmtDataBlock(pBlocks, true)); @@ -279,7 +279,7 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool freeRequest) { } pStmt->exec.autoCreateTbl = false; - + if (keepTable) { return TSDB_CODE_SUCCESS; } @@ -320,13 +320,15 @@ int32_t stmtCleanSQLInfo(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } -int32_t stmtRebuildDataBlock(STscStmt* pStmt, STableDataBlocks *pDataBlock, STableDataBlocks **newBlock, uint64_t uid) { - SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp); +int32_t stmtRebuildDataBlock(STscStmt* pStmt, STableDataBlocks* pDataBlock, STableDataBlocks** newBlock, uint64_t uid) { + SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp); SVgroupInfo vgInfo = {0}; - - STMT_ERR_RET(catalogGetTableHashVgroup(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->bInfo.sname, &vgInfo)); - STMT_ERR_RET(taosHashPut(pStmt->exec.pVgHash, (const char*)&vgInfo.vgId, sizeof(vgInfo.vgId), (char*)&vgInfo, sizeof(vgInfo))); - + + STMT_ERR_RET(catalogGetTableHashVgroup(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->bInfo.sname, + &vgInfo)); + STMT_ERR_RET( + taosHashPut(pStmt->exec.pVgHash, (const char*)&vgInfo.vgId, sizeof(vgInfo.vgId), (char*)&vgInfo, sizeof(vgInfo))); + STMT_ERR_RET(qRebuildStmtDataBlock(newBlock, pDataBlock, uid, vgInfo.vgId)); return TSDB_CODE_SUCCESS; @@ -335,8 +337,9 @@ int32_t stmtRebuildDataBlock(STscStmt* pStmt, STableDataBlocks *pDataBlock, STab int32_t stmtGetFromCache(STscStmt* pStmt) { pStmt->bInfo.needParse = true; pStmt->bInfo.inExecCache = false; - - STableDataBlocks *pBlockInExec = taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + + STableDataBlocks* pBlockInExec = + taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (pBlockInExec) { pStmt->bInfo.needParse = false; pStmt->bInfo.inExecCache = true; @@ -352,7 +355,7 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { pStmt->bInfo.needParse = false; return TSDB_CODE_SUCCESS; } - + return TSDB_CODE_SUCCESS; } @@ -367,24 +370,25 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { pStmt->exec.autoCreateTbl = true; pStmt->bInfo.tbUid = 0; - + STableDataBlocks* pNewBlock = NULL; STMT_ERR_RET(stmtRebuildDataBlock(pStmt, pCache->pDataBlock, &pNewBlock, 0)); - - if (taosHashPut(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName), &pNewBlock, POINTER_BYTES)) { + + if (taosHashPut(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName), &pNewBlock, + POINTER_BYTES)) { STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - + return TSDB_CODE_SUCCESS; } - + STMT_RET(stmtCleanBindInfo(pStmt)); } - - STableMeta *pTableMeta = NULL; - SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp); - int32_t code = catalogGetTableMeta(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->bInfo.sname, &pTableMeta); + STableMeta* pTableMeta = NULL; + SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp); + int32_t code = + catalogGetTableMeta(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->bInfo.sname, &pTableMeta); if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code) { STMT_ERR_RET(stmtCleanBindInfo(pStmt)); @@ -398,7 +402,7 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { int8_t tableType = pTableMeta->tableType; taosMemoryFree(pTableMeta); uint64_t cacheUid = (TSDB_CHILD_TABLE == tableType) ? suid : uid; - + if (uid == pStmt->bInfo.tbUid) { pStmt->bInfo.needParse = false; @@ -408,8 +412,9 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { if (pStmt->bInfo.inExecCache) { SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &cacheUid, sizeof(cacheUid)); if (NULL == pCache) { - tscError("table [%s, %" PRIx64 ", %" PRIx64 "] found in exec blockHash, but not in sql blockHash", pStmt->bInfo.tbFName, uid, cacheUid); - + tscError("table [%s, %" PRIx64 ", %" PRIx64 "] found in exec blockHash, but not in sql blockHash", + pStmt->bInfo.tbFName, uid, cacheUid); + STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR); } @@ -437,7 +442,8 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { STableDataBlocks* pNewBlock = NULL; STMT_ERR_RET(stmtRebuildDataBlock(pStmt, pCache->pDataBlock, &pNewBlock, uid)); - if (taosHashPut(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName), &pNewBlock, POINTER_BYTES)) { + if (taosHashPut(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName), &pNewBlock, + POINTER_BYTES)) { STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -521,10 +527,11 @@ int stmtSetTbName(TAOS_STMT* stmt, const char* tbName) { if (NULL == pStmt->exec.pRequest) { STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest)); } - - STMT_ERR_RET(qCreateSName(&pStmt->bInfo.sname, tbName, pStmt->taos->acctId, pStmt->exec.pRequest->pDb, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); + + STMT_ERR_RET(qCreateSName(&pStmt->bInfo.sname, tbName, pStmt->taos->acctId, pStmt->exec.pRequest->pDb, + pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); tNameExtractFullName(&pStmt->bInfo.sname, pStmt->bInfo.tbFName); - + STMT_ERR_RET(stmtGetFromCache(pStmt)); if (pStmt->bInfo.needParse) { @@ -548,7 +555,8 @@ int stmtSetTbTags(TAOS_STMT* stmt, TAOS_MULTI_BIND* tags) { return TSDB_CODE_SUCCESS; } - STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + STableDataBlocks** pDataBlock = + (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pDataBlock) { tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); @@ -566,7 +574,8 @@ int32_t stmtFetchTagFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD** fiel STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); } - STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + STableDataBlocks** pDataBlock = + (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pDataBlock) { tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); @@ -583,7 +592,8 @@ int32_t stmtFetchColFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD** fiel STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); } - STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + STableDataBlocks** pDataBlock = + (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pDataBlock) { tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); @@ -618,8 +628,8 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { } if (STMT_TYPE_QUERY == pStmt->sql.type) { - STMT_ERR_RET(qStmtBindParams(pStmt->sql.pQuery, bind, colIdx, pStmt->exec.pRequest->requestId)); - + STMT_ERR_RET(qStmtBindParams(pStmt->sql.pQuery, bind, colIdx)); + SParseContext ctx = {.requestId = pStmt->exec.pRequest->requestId, .acctId = pStmt->taos->acctId, .db = pStmt->exec.pRequest->pDb, @@ -633,27 +643,29 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { .pUser = pStmt->taos->user}; ctx.mgmtEpSet = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp); STMT_ERR_RET(catalogGetHandle(pStmt->taos->pAppInfo->clusterId, &ctx.pCatalog)); - + STMT_ERR_RET(qStmtParseQuerySql(&ctx, pStmt->sql.pQuery)); if (pStmt->sql.pQuery->haveResultSet) { - setResSchemaInfo(&pStmt->exec.pRequest->body.resInfo, pStmt->sql.pQuery->pResSchema, pStmt->sql.pQuery->numOfResCols); + setResSchemaInfo(&pStmt->exec.pRequest->body.resInfo, pStmt->sql.pQuery->pResSchema, + pStmt->sql.pQuery->numOfResCols); setResPrecision(&pStmt->exec.pRequest->body.resInfo, pStmt->sql.pQuery->precision); } - + TSWAP(pStmt->exec.pRequest->dbList, pStmt->sql.pQuery->pDbList); - TSWAP(pStmt->exec.pRequest->tableList, pStmt->sql.pQuery->pTableList); + TSWAP(pStmt->exec.pRequest->tableList, pStmt->sql.pQuery->pTableList); - //if (STMT_TYPE_QUERY == pStmt->sql.queryRes) { - // STMT_ERR_RET(stmtRestoreQueryFields(pStmt)); - //} + // if (STMT_TYPE_QUERY == pStmt->sql.queryRes) { + // STMT_ERR_RET(stmtRestoreQueryFields(pStmt)); + // } - //STMT_ERR_RET(stmtBackupQueryFields(pStmt)); + // STMT_ERR_RET(stmtBackupQueryFields(pStmt)); return TSDB_CODE_SUCCESS; } - - STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + + STableDataBlocks** pDataBlock = + (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pDataBlock) { tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); @@ -694,19 +706,19 @@ int stmtAddBatch(TAOS_STMT* stmt) { return TSDB_CODE_SUCCESS; } -int stmtUpdateTableUid(STscStmt* pStmt, SSubmitRsp *pRsp) { +int stmtUpdateTableUid(STscStmt* pStmt, SSubmitRsp* pRsp) { if (pRsp->nBlocks <= 0) { tscError("invalid submit resp block number %d", pRsp->nBlocks); STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR); } - size_t keyLen = 0; - STableDataBlocks **pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL); + size_t keyLen = 0; + STableDataBlocks** pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL); while (pIter) { - STableDataBlocks *pBlock = *pIter; - char *key = taosHashGetKey(pIter, &keyLen); - - STableMeta *pMeta = qGetTableMetaInDataBlock(pBlock); + STableDataBlocks* pBlock = *pIter; + char* key = taosHashGetKey(pIter, &keyLen); + + STableMeta* pMeta = qGetTableMetaInDataBlock(pBlock); if (pMeta->uid != pStmt->bInfo.tbUid) { tscError("table uid %" PRIx64 " mis-match with current table uid %" PRIx64, pMeta->uid, pStmt->bInfo.tbUid); STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR); @@ -717,24 +729,25 @@ int stmtUpdateTableUid(STscStmt* pStmt, SSubmitRsp *pRsp) { continue; } - SSubmitBlkRsp *blkRsp = NULL; - int32_t i = 0; + SSubmitBlkRsp* blkRsp = NULL; + int32_t i = 0; for (; i < pRsp->nBlocks; ++i) { blkRsp = pRsp->pBlocks + i; if (strlen(blkRsp->tblFName) != keyLen) { continue; } - + if (strncmp(blkRsp->tblFName, key, keyLen)) { continue; } - + break; } if (i < pRsp->nBlocks) { - tscDebug("auto created table %s uid updated from %" PRIx64 " to %" PRIx64, blkRsp->tblFName, pMeta->uid, blkRsp->uid); - + tscDebug("auto created table %s uid updated from %" PRIx64 " to %" PRIx64, blkRsp->tblFName, pMeta->uid, + blkRsp->uid); + pMeta->uid = blkRsp->uid; pStmt->bInfo.tbUid = blkRsp->uid; } else { @@ -748,11 +761,11 @@ int stmtUpdateTableUid(STscStmt* pStmt, SSubmitRsp *pRsp) { return TSDB_CODE_SUCCESS; } -int stmtExec(TAOS_STMT *stmt) { - STscStmt* pStmt = (STscStmt*)stmt; - int32_t code = 0; - SSubmitRsp *pRsp = NULL; - bool autoCreateTbl = pStmt->exec.autoCreateTbl; +int stmtExec(TAOS_STMT* stmt) { + STscStmt* pStmt = (STscStmt*)stmt; + int32_t code = 0; + SSubmitRsp* pRsp = NULL; + bool autoCreateTbl = pStmt->exec.autoCreateTbl; STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_EXECUTE)); @@ -760,7 +773,8 @@ int stmtExec(TAOS_STMT *stmt) { launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, TSDB_CODE_SUCCESS, true, NULL); } else { STMT_ERR_RET(qBuildStmtOutput(pStmt->sql.pQuery, pStmt->exec.pVgHash, pStmt->exec.pBlockHash)); - launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, TSDB_CODE_SUCCESS, true, (autoCreateTbl ? (void**)&pRsp : NULL)); + launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, TSDB_CODE_SUCCESS, true, + (autoCreateTbl ? (void**)&pRsp : NULL)); } if (pStmt->exec.pRequest->code && NEED_CLIENT_HANDLE_ERROR(pStmt->exec.pRequest->code)) { @@ -787,10 +801,10 @@ _return: tscError("no submit resp got for auto create table"); STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR); } - + STMT_ERR_RET(stmtUpdateTableUid(pStmt, pRsp)); } - + ++pStmt->sql.runTimes; STMT_RET(code); diff --git a/source/client/test/CMakeLists.txt b/source/client/test/CMakeLists.txt index 3fc91d07b6..bb9f0ed23e 100644 --- a/source/client/test/CMakeLists.txt +++ b/source/client/test/CMakeLists.txt @@ -41,3 +41,7 @@ TARGET_INCLUDE_DIRECTORIES( PRIVATE "${TD_SOURCE_DIR}/source/client/inc" ) +#add_test( +# NAME smlTest +# COMMAND smlTest +#) diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index e567a0c3e8..eeab844712 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -207,7 +207,7 @@ TEST(testCase, smlParseCols_Error_Test) { char *sql = (char*)taosMemoryCalloc(256, 1); memcpy(sql, data[i], len + 1); SArray *cols = taosArrayInit(8, POINTER_BYTES); - int32_t ret = smlParseCols(sql, len, cols, false, dumplicateKey, &msgBuf); + int32_t ret = smlParseCols(sql, len, cols, NULL, false, dumplicateKey, &msgBuf); ASSERT_NE(ret, TSDB_CODE_SUCCESS); taosHashClear(dumplicateKey); taosMemoryFree(sql); @@ -233,7 +233,7 @@ TEST(testCase, smlParseCols_tag_Test) { const char *data = "cbin=\"passit helloc\",cnch=L\"iisdfsf\",cbool=false,cf64=4.31f64,cf64_=8.32,cf32=8.23f32,ci8=-34i8,cu8=89u8,ci16=233i16,cu16=898u16,ci32=98289i32,cu32=12323u32,ci64=-89238i64,ci=989i,cu64=8989323u64,cbooltrue=true,cboolt=t,cboolf=f,cnch_=l\"iuwq\""; int32_t len = strlen(data); - int32_t ret = smlParseCols(data, len, cols, true, dumplicateKey, &msgBuf); + int32_t ret = smlParseCols(data, len, cols, NULL, true, dumplicateKey, &msgBuf); ASSERT_EQ(ret, TSDB_CODE_SUCCESS); int32_t size = taosArrayGetSize(cols); ASSERT_EQ(size, 19); @@ -265,7 +265,7 @@ TEST(testCase, smlParseCols_tag_Test) { len = 0; memset(msgBuf.buf, 0, msgBuf.len); taosHashClear(dumplicateKey); - ret = smlParseCols(data, len, cols, true, dumplicateKey, &msgBuf); + ret = smlParseCols(data, len, cols, NULL, true, dumplicateKey, &msgBuf); ASSERT_EQ(ret, TSDB_CODE_SUCCESS); size = taosArrayGetSize(cols); ASSERT_EQ(size, 1); @@ -298,7 +298,7 @@ TEST(testCase, smlParseCols_Test) { int32_t len = strlen(data); char *sql = (char*)taosMemoryCalloc(1024, 1); memcpy(sql, data, len + 1); - int32_t ret = smlParseCols(sql, len, cols, false, dumplicateKey, &msgBuf); + int32_t ret = smlParseCols(sql, len, cols, NULL, false, dumplicateKey, &msgBuf); ASSERT_EQ(ret, TSDB_CODE_SUCCESS); int32_t size = taosArrayGetSize(cols); ASSERT_EQ(size, 19); @@ -488,44 +488,118 @@ TEST(testCase, smlProcess_influx_Test) { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(taos, nullptr); - TAOS_RES* pRes = taos_query(taos, "create database if not exists sml_db"); + TAOS_RES* pRes = taos_query(taos, "create database if not exists inflx_db"); taos_free_result(pRes); - pRes = taos_query(taos, "use sml_db"); + pRes = taos_query(taos, "use inflx_db"); taos_free_result(pRes); SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); - SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true); + SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); ASSERT_NE(info, nullptr); const char *sql[] = { - "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 load_capacity=1500,fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=124,velocity=0,heading=221,grade=0 1451606400000000000", - "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 load_capacity=1500,fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=124,velocity=0,heading=221,grade=0,fuel_consumption=25 1451607400000000000", - "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 load_capacity=1500,fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=124,heading=221,grade=0,fuel_consumption=25 1451608400000000000", - "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=124,velocity=0,heading=221,grade=0,fuel_consumption=25 1451609400000000000", - "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 fuel_consumption=25,grade=0 1451619400000000000", - "readings,name=truck_1,fleet=South,driver=Albert,model=F-150,device_version=v1.5 load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=72.45258,longitude=68.83761,elevation=255,velocity=0,heading=181,grade=0,fuel_consumption=25 1451606400000000000", - "readings,name=truck_2,driver=Derek,model=F-150,device_version=v1.5 load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=24.5208,longitude=28.09377,elevation=428,velocity=0,heading=304,grade=0,fuel_consumption=25 1451606400000000000", - "readings,name=truck_2,fleet=North,driver=Derek,model=F-150 load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=24.5208,longitude=28.09377,elevation=428,velocity=0,heading=304,grade=0,fuel_consumption=25 1451609400000000000", - "readings,fleet=South,name=truck_0,driver=Trish,model=H-2,device_version=v2.3 fuel_consumption=25,grade=0 1451629400000000000", - "stable,t1=t1,t2=t2,t3=t3 c1=1,c2=2,c3=3,c4=4 1451629500000000000", - "stable,t2=t2,t1=t1,t3=t3 c1=1,c3=3,c4=4 1451629600000000000", + "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 load_capacity=1500,fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=124,velocity=0,heading=221,grade=0 1451606401000000000", + "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 load_capacity=1500,fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=124,velocity=0,heading=221,grade=0,fuel_consumption=25 1451607402000000000", + "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 load_capacity=1500,fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=124,heading=221,grade=0,fuel_consumption=25 1451608403000000000", + "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=124,velocity=0,heading=221,grade=0,fuel_consumption=25 1451609404000000000", + "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 fuel_consumption=25,grade=0 1451619405000000000", + "readings,name=truck_1,fleet=South,driver=Albert,model=F-150,device_version=v1.5 load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=72.45258,longitude=68.83761,elevation=255,velocity=0,heading=181,grade=0,fuel_consumption=25 145160640600000000", + "readings,name=truck_2,driver=Derek,model=F-150,device_version=v1.5 load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=24.5208,longitude=28.09377,elevation=428,velocity=0,heading=304,grade=0,fuel_consumption=25 1451606407000000000", + "readings,name=truck_2,fleet=North,driver=Derek,model=F-150 load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=24.5208,longitude=28.09377,elevation=428,velocity=0,heading=304,grade=0,fuel_consumption=25 1451609408000000000", + "readings,fleet=South,name=truck_0,driver=Trish,model=H-2,device_version=v2.3 fuel_consumption=25,grade=0 1451629409000000000", + "stable,t1=t1,t2=t2,t3=t3 c1=1,c2=2,c3=\"kk\",c4=4 1451629501000000000", + "stable,t2=t2,t1=t1,t3=t3 c1=1,c3=\"\",c4=4 1451629602000000000", }; int ret = smlProcess(info, (char**)sql, sizeof(sql)/sizeof(sql[0])); ASSERT_EQ(ret, 0); -// TAOS_RES *res = taos_query(taos, "select * from t_6885c584b98481584ee13dac399e173d"); -// ASSERT_NE(res, nullptr); -// int fieldNum = taos_field_count(res); -// ASSERT_EQ(fieldNum, 5); -// int rowNum = taos_affected_rows(res); -// ASSERT_EQ(rowNum, 2); -// for (int i = 0; i < rowNum; ++i) { -// TAOS_ROW rows = taos_fetch_row(res); -// } -// taos_free_result(res); + // case 1 + TAOS_RES *res = taos_query(taos, "select * from t_91e0b182be80332b5c530cbf872f760e"); + ASSERT_NE(res, nullptr); + int fieldNum = taos_field_count(res); + ASSERT_EQ(fieldNum, 11); + printf("fieldNum:%d\n", fieldNum); + + TAOS_ROW row = NULL; + int32_t rowIndex = 0; + while((row = taos_fetch_row(res)) != NULL) { + int64_t ts = *(int64_t*)row[0]; + double load_capacity = *(double*)row[1]; + double fuel_capacity = *(double*)row[2]; + double nominal_fuel_consumption = *(double*)row[3]; + double latitude = *(double*)row[4]; + double longitude = *(double*)row[5]; + double elevation = *(double*)row[6]; + double velocity = *(double*)row[7]; + double heading = *(double*)row[8]; + double grade = *(double*)row[9]; + double fuel_consumption = *(double*)row[10]; + if(rowIndex == 0){ + ASSERT_EQ(ts, 1451606407000); + ASSERT_EQ(load_capacity, 2000); + ASSERT_EQ(fuel_capacity, 200); + ASSERT_EQ(nominal_fuel_consumption, 15); + ASSERT_EQ(latitude, 24.5208); + ASSERT_EQ(longitude, 28.09377); + ASSERT_EQ(elevation, 428); + ASSERT_EQ(velocity, 0); + ASSERT_EQ(heading, 304); + ASSERT_EQ(grade, 0); + ASSERT_EQ(fuel_consumption, 25); + }else{ + ASSERT_FALSE(1); + } + rowIndex++; + } + taos_free_result(res); + + // case 2 + res = taos_query(taos, "select * from t_6885c584b98481584ee13dac399e173d"); + ASSERT_NE(res, nullptr); + fieldNum = taos_field_count(res); + ASSERT_EQ(fieldNum, 5); + printf("fieldNum:%d\n", fieldNum); + + rowIndex = 0; + while((row = taos_fetch_row(res)) != NULL) { + int *length = taos_fetch_lengths(res); + + int64_t ts = *(int64_t*)row[0]; + double c1 = *(double*)row[1]; + double c4 = *(double*)row[4]; + if(rowIndex == 0){ + ASSERT_EQ(ts, 1451629501000); + ASSERT_EQ(c1, 1); + ASSERT_EQ(*(double*)row[2], 2); + ASSERT_EQ(length[3], 2); + ASSERT_EQ(memcmp(row[3], "kk", length[3]), 0); + ASSERT_EQ(c4, 4); + }else if(rowIndex == 1){ + ASSERT_EQ(ts, 1451629602000); + ASSERT_EQ(c1, 1); + ASSERT_EQ(row[2], nullptr); + ASSERT_EQ(length[3], 0); + ASSERT_EQ(c4, 4); + }else{ + ASSERT_FALSE(1); + } + rowIndex++; + } + taos_free_result(res); + + // case 2 + res = taos_query(taos, "show tables"); + ASSERT_NE(res, nullptr); + + row = taos_fetch_row(res); + int rowNum = taos_affected_rows(res); + ASSERT_EQ(rowNum, 5); + taos_free_result(res); + + destroyRequest(request); smlDestroyInfo(info); } @@ -544,7 +618,7 @@ TEST(testCase, smlParseLine_error_Test) { SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); - SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true); + SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); ASSERT_NE(info, nullptr); const char *sql[] = { @@ -584,16 +658,16 @@ TEST(testCase, smlProcess_telnet_Test) { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(taos, nullptr); - TAOS_RES* pRes = taos_query(taos, "create database if not exists sml_db"); + TAOS_RES* pRes = taos_query(taos, "create database if not exists telnet_db"); taos_free_result(pRes); - pRes = taos_query(taos, "use sml_db"); + pRes = taos_query(taos, "use telnet_db"); taos_free_result(pRes); SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); - SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true); + SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); ASSERT_NE(info, nullptr); const char *sql[] = { @@ -605,27 +679,31 @@ TEST(testCase, smlProcess_telnet_Test) { int ret = smlProcess(info, (char**)sql, sizeof(sql)/sizeof(sql[0])); ASSERT_EQ(ret, 0); -// TAOS_RES *res = taos_query(taos, "select * from t_8c30283b3c4131a071d1e16cf6d7094a"); -// ASSERT_NE(res, nullptr); -// int fieldNum = taos_field_count(res); -// ASSERT_EQ(fieldNum, 2); -// int rowNum = taos_affected_rows(res); -// ASSERT_EQ(rowNum, 1); -// for (int i = 0; i < rowNum; ++i) { -// TAOS_ROW rows = taos_fetch_row(res); -// } -// taos_free_result(res); + // case 1 + TAOS_RES *res = taos_query(taos, "select * from t_8c30283b3c4131a071d1e16cf6d7094a"); + ASSERT_NE(res, nullptr); + int fieldNum = taos_field_count(res); + ASSERT_EQ(fieldNum, 2); + + TAOS_ROW row = taos_fetch_row(res); + int64_t ts = *(int64_t*)row[0]; + double c1 = *(double*)row[1]; + ASSERT_EQ(ts, 1479496100000); + ASSERT_EQ(c1, 42); + + int rowNum = taos_affected_rows(res); + ASSERT_EQ(rowNum, 1); + taos_free_result(res); + + // case 2 + res = taos_query(taos, "show tables"); + ASSERT_NE(res, nullptr); + + row = taos_fetch_row(res); + rowNum = taos_affected_rows(res); + ASSERT_EQ(rowNum, 3); + taos_free_result(res); -// res = taos_query(taos, "select * from t_6931529054e5637ca92c78a1ad441961"); -// ASSERT_NE(res, nullptr); -// fieldNum = taos_field_count(res); -// ASSERT_EQ(fieldNum, 2); -// rowNum = taos_affected_rows(res); -// ASSERT_EQ(rowNum, 2); -// for (int i = 0; i < rowNum; ++i) { -// TAOS_ROW rows = taos_fetch_row(res); -// } -// taos_free_result(res); destroyRequest(request); smlDestroyInfo(info); } @@ -634,16 +712,16 @@ TEST(testCase, smlProcess_json1_Test) { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(taos, nullptr); - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db"); + TAOS_RES *pRes = taos_query(taos, "create database if not exists json_db"); taos_free_result(pRes); - pRes = taos_query(taos, "use sml_db"); + pRes = taos_query(taos, "use json_db"); taos_free_result(pRes); SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, NULL, NULL, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); - SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true); + SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); ASSERT_NE(info, nullptr); const char *sql = @@ -670,16 +748,31 @@ TEST(testCase, smlProcess_json1_Test) { int ret = smlProcess(info, (char **)(&sql), -1); ASSERT_EQ(ret, 0); -// TAOS_RES *res = taos_query(taos, "select * from t_cb27a7198d637b4f1c6464bd73f756a7"); -// ASSERT_NE(res, nullptr); -// int fieldNum = taos_field_count(res); -// ASSERT_EQ(fieldNum, 2); - // int rowNum = taos_affected_rows(res); - // ASSERT_EQ(rowNum, 1); - // for (int i = 0; i < rowNum; ++i) { - // TAOS_ROW rows = taos_fetch_row(res); - // } -// taos_free_result(res); + // case 1 + TAOS_RES *res = taos_query(taos, "select * from t_cb27a7198d637b4f1c6464bd73f756a7"); + ASSERT_NE(res, nullptr); + int fieldNum = taos_field_count(res); + ASSERT_EQ(fieldNum, 2); + + TAOS_ROW row = taos_fetch_row(res); + int64_t ts = *(int64_t*)row[0]; + double c1 = *(double*)row[1]; + ASSERT_EQ(ts, 1346846400000); + ASSERT_EQ(c1, 18); + + int rowNum = taos_affected_rows(res); + ASSERT_EQ(rowNum, 1); + taos_free_result(res); + + // case 2 + res = taos_query(taos, "show tables"); + ASSERT_NE(res, nullptr); + + row = taos_fetch_row(res); + rowNum = taos_affected_rows(res); + ASSERT_EQ(rowNum, 2); + taos_free_result(res); + destroyRequest(request); smlDestroyInfo(info); } @@ -697,7 +790,7 @@ TEST(testCase, smlProcess_json2_Test) { SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, NULL, NULL, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); - SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true); + SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); ASSERT_NE(info, nullptr); const char *sql = "{\n" @@ -741,7 +834,7 @@ TEST(testCase, smlProcess_json3_Test) { SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, NULL, NULL, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); - SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true); + SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); ASSERT_NE(info, nullptr); const char *sql = "{\n" @@ -813,7 +906,7 @@ TEST(testCase, smlProcess_json4_Test) { SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); - SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true); + SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); ASSERT_NE(info, nullptr); const char *sql = "{\n" " \"metric\": \"meter_current2\",\n" @@ -875,7 +968,7 @@ TEST(testCase, smlParseTelnetLine_error_Test) { SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); - SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true); + SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); ASSERT_NE(info, nullptr); int32_t ret = 0; @@ -924,7 +1017,7 @@ TEST(testCase, smlParseTelnetLine_diff_type_Test) { SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); - SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true); + SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); ASSERT_NE(info, nullptr); const char *sql[2] = { @@ -951,7 +1044,7 @@ TEST(testCase, smlParseTelnetLine_json_error_Test) { SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); - SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true); + SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); ASSERT_NE(info, nullptr); int32_t ret = 0; @@ -1019,7 +1112,7 @@ TEST(testCase, smlParseTelnetLine_diff_json_type1_Test) { SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); - SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true); + SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); ASSERT_NE(info, nullptr); const char *sql[2] = { @@ -1064,7 +1157,7 @@ TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) { SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); - SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true); + SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); ASSERT_NE(info, nullptr); const char *sql[2] = { diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index aaad606feb..f831db5457 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -76,6 +76,11 @@ int32_t tsTelemInterval = 86400; char tsTelemServer[TSDB_FQDN_LEN] = "telemetry.taosdata.com"; uint16_t tsTelemPort = 80; +// schemaless +char tsSmlChildTableName[TSDB_TABLE_NAME_LEN] = ""; //user defined child table name can be specified in tag value. + //If set to empty system will generate table name using MD5 hash. +bool tsSmlDataFormat = true; // true means that the name and order of cols in each line are the same(only for influx protocol) + // query int32_t tsQueryPolicy = 1; @@ -319,6 +324,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "compressColData", tsCompressColData, -1, 100000000, 1) != 0) return -1; if (cfgAddBool(pCfg, "keepColumnName", tsKeepOriginalColumnName, 1) != 0) return -1; if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 3, 1) != 0) return -1; + if (cfgAddString(pCfg, "smlChildTableName", "", 1) != 0) return -1; + if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1; tsNumOfTaskQueueThreads = tsNumOfCores / 4; tsNumOfTaskQueueThreads = TRANGE(tsNumOfTaskQueueThreads, 1, 2); @@ -433,7 +440,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsRpcQueueMemoryAllowed = tsTotalMemoryKB * 1024 * 0.1; tsRpcQueueMemoryAllowed = TRANGE(tsRpcQueueMemoryAllowed, TSDB_MAX_WAL_SIZE * 10L, TSDB_MAX_WAL_SIZE * 10000L); - if (cfgAddInt64(pCfg, "rpcQueueMemoryAllowed", tsRpcQueueMemoryAllowed, 1, INT64_MAX, 0) != 0) return -1; + if (cfgAddInt64(pCfg, "rpcQueueMemoryAllowed", tsRpcQueueMemoryAllowed, TSDB_MAX_WAL_SIZE * 10L, INT64_MAX, 0) != 0) + return -1; if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, 0) != 0) return -1; if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, 0) != 0) return -1; @@ -512,6 +520,9 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { return -1; } + tstrncpy(tsSmlChildTableName, cfgGetItem(pCfg, "smlChildTableName")->str, TSDB_TABLE_NAME_LEN); + tsSmlDataFormat = cfgGetItem(pCfg, "smlDataFormat")->bval; + tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32; tsCompressMsgSize = cfgGetItem(pCfg, "compressMsgSize")->i32; tsCompressColData = cfgGetItem(pCfg, "compressColData")->i32; diff --git a/source/dnode/mgmt/test/sut/src/sut.cpp b/source/dnode/mgmt/test/sut/src/sut.cpp index cc32f047af..7d37422b5a 100644 --- a/source/dnode/mgmt/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/test/sut/src/sut.cpp @@ -30,6 +30,7 @@ void Testbase::InitLog(const char* path) { tsdbDebugFlag = 0; tsLogEmbedded = 1; tsAsyncLog = 0; + tsRpcQueueMemoryAllowed = 1024 * 1024 * 64; taosRemoveDir(path); taosMkDir(path); @@ -82,7 +83,7 @@ SRpcMsg* Testbase::SendReq(tmsg_t msgType, void* pCont, int32_t contLen) { return client.SendReq(&rpcMsg); } -int32_t Testbase::SendShowReq(int8_t showType, const char *tb, const char* db) { +int32_t Testbase::SendShowReq(int8_t showType, const char* tb, const char* db) { if (showRsp != NULL) { rpcFreeCont(showRsp); showRsp = NULL; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 5a834f4d33..faa0d71d3b 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -386,7 +386,7 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt req.schema.sver = pStb->version; req.schema.pSchema = pStb->pColumns; req.schemaTag.nCols = pStb->numOfTags; - req.schemaTag.nCols = 0; + req.schemaTag.sver = 1; req.schemaTag.pSchema = pStb->pTags; if (req.rollup) { diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 6bfcc34c85..9064085b2b 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -420,7 +420,8 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl // get table entry SDecoder dc = {0}; tDecoderInit(&dc, pData, nData); - metaDecodeEntry(&dc, &entry); + ret = metaDecodeEntry(&dc, &entry); + ASSERT(ret == 0); if (entry.type != TSDB_NORMAL_TABLE) { terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; @@ -468,11 +469,11 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl goto _err; } pSchema->sver++; - pSchema->nCols--; tlen = (pSchema->nCols - iCol - 1) * sizeof(SSchema); if (tlen) { memmove(pColumn, pColumn + 1, tlen); } + pSchema->nCols--; break; case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: if (pColumn == NULL) { @@ -598,10 +599,41 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA goto _err; } - { - // TODO: + if (iCol == 0) { + // TODO : need to update tag index } + ctbEntry.version = version; + SKVRowBuilder kvrb = {0}; + const SKVRow pOldTag = (const SKVRow)ctbEntry.ctbEntry.pTags; + SKVRow pNewTag = NULL; + + tdInitKVRowBuilder(&kvrb); + for (int32_t i = 0; i < pTagSchema->nCols; i++) { + SSchema *pCol = &pTagSchema->pSchema[i]; + if (iCol == i) { + tdAddColToKVRow(&kvrb, pCol->colId, pAlterTbReq->pTagVal, pAlterTbReq->nTagVal); + } else { + void *p = tdGetKVRowValOfCol(pOldTag, pCol->colId); + if (p) { + if (IS_VAR_DATA_TYPE(pCol->type)) { + tdAddColToKVRow(&kvrb, pCol->colId, p, varDataTLen(p)); + } else { + tdAddColToKVRow(&kvrb, pCol->colId, p, pCol->bytes); + } + } + } + } + + ctbEntry.ctbEntry.pTags = tdGetKVRowFromBuilder(&kvrb); + tdDestroyKVRowBuilder(&kvrb); + + // save to table.db + metaSaveToTbDb(pMeta, &ctbEntry); + + // save to uid.idx + tdbDbUpsert(pMeta->pUidIdx, &ctbEntry.uid, sizeof(tb_uid_t), &version, sizeof(version), &pMeta->txn); + if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); tdbDbcClose(pTbDbc); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index c1be8db473..254d452cb3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3055,7 +3055,8 @@ static bool loadDataBlockFromTableSeq(STsdbReadHandle* pTsdbReadHandle) { bool tsdbNextDataBlock(tsdbReaderT pHandle) { STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*)pHandle; - for (int32_t i = 0; i < taosArrayGetSize(pTsdbReadHandle->pColumns); ++i) { + size_t numOfCols = taosArrayGetSize(pTsdbReadHandle->pColumns); + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); colInfoDataCleanup(pColInfo, pTsdbReadHandle->outputCapacity); } diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index e62729a051..edc21626f5 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -139,7 +139,7 @@ typedef struct { int32_t colId; } SStddevInterResult; -void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, bool sortGroupResult); +void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int32_t order); void initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList); void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 6100d416b1..bf178612ba 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -388,13 +388,15 @@ typedef struct SStreamBlockScanInfo { SColumnInfo* pCols; // the output column info uint64_t numOfRows; // total scanned rows uint64_t numOfExec; // execution times - void* readerHandle; // stream block reader handle + void* streamBlockReader;// stream block reader handle SArray* pColMatchInfo; // SNode* pCondition; SArray* tsArray; SUpdateInfo* pUpdateInfo; int32_t primaryTsIndex; // primary time stamp slot id void* pDataReader; + SReadHandle readHandle; + uint64_t tableUid; // queried super table uid EStreamScanMode scanMode; SOperatorInfo* pOperatorDumy; SInterval interval; // if the upstream is an interval operator, the interval info is also kept here. @@ -706,9 +708,10 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx SExprInfo* pScalarExprInfo, int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, void* pDataReader, SSDataBlock* pResBlock, - SArray* pColList, SArray* pTableIdList, SExecTaskInfo* pTaskInfo, - SNode* pConditions, SOperatorInfo* pOperatorDumy); +SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, void* pDataReader, SReadHandle* pHandle, + uint64_t uid, SSDataBlock* pResBlock, SArray* pColList, + SArray* pTableIdList, SExecTaskInfo* pTaskInfo, SNode* pCondition, + SOperatorInfo* pOperatorDumy); SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SInterval* pInterval, STimeWindow* pWindow, SSDataBlock* pResBlock, int32_t fillType, SNodeListNode* fillVal, diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 763dcef790..14cff488ec 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -184,7 +184,7 @@ void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo) { pGroupResInfo->index = 0; } -static int32_t resultrowCompar1(const void* p1, const void* p2) { +static int32_t resultrowComparAsc(const void* p1, const void* p2) { SResKeyPos* pp1 = *(SResKeyPos**) p1; SResKeyPos* pp2 = *(SResKeyPos**) p2; @@ -202,7 +202,11 @@ static int32_t resultrowCompar1(const void* p1, const void* p2) { } } -void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, bool sortGroupResult) { +static int32_t resultrowComparDesc(const void* p1, const void* p2) { + return resultrowComparAsc(p2, p1); +} + +void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int32_t order) { if (pGroupResInfo->pRows != NULL) { taosArrayDestroy(pGroupResInfo->pRows); } @@ -224,8 +228,9 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, boo taosArrayPush(pGroupResInfo->pRows, &p); } - if (sortGroupResult) { - qsort(pGroupResInfo->pRows->pData, taosArrayGetSize(pGroupResInfo->pRows), POINTER_BYTES, resultrowCompar1); + if (order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC) { + __compar_fn_t fn = (order == TSDB_ORDER_ASC)? resultrowComparAsc:resultrowComparDesc; + qsort(pGroupResInfo->pRows->pData, taosArrayGetSize(pGroupResInfo->pRows), POINTER_BYTES, fn); } pGroupResInfo->index = 0; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 4863b03fb9..320450eb6e 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -14,6 +14,7 @@ */ #include "executor.h" +#include #include "executorimpl.h" #include "planner.h" #include "tdatablock.h" @@ -46,7 +47,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu } if (type == STREAM_DATA_TYPE_SUBMIT_BLOCK) { - if (tqReadHandleSetMsg(pInfo->readerHandle, input, 0) < 0) { + if (tqReadHandleSetMsg(pInfo->streamBlockReader, input, 0) < 0) { qError("submit msg messed up when initing stream block, %s" PRIx64, id); return TSDB_CODE_QRY_APP_ERROR; } @@ -128,7 +129,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) { int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isAdd) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - // traverse to the streamscan node to add this table id + // traverse to the stream scanner node to add this table id SOperatorInfo* pInfo = pTaskInfo->pRoot; while (pInfo->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { pInfo = pInfo->pDownstream[0]; @@ -136,7 +137,31 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isA SStreamBlockScanInfo* pScanInfo = pInfo->info; if (isAdd) { - int32_t code = tqReadHandleAddTbUidList(pScanInfo->readerHandle, tableIdList); + SArray* qa = taosArrayInit(4, sizeof(tb_uid_t)); + + SMetaReader mr = {0}; + metaReaderInit(&mr, pScanInfo->readHandle.meta, 0); + for(int32_t i = 0; i < taosArrayGetSize(tableIdList); ++i) { + int64_t* id = (int64_t*)taosArrayGet(tableIdList, i); + + int32_t code = metaGetTableEntryByUid(&mr, *id); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get table meta, uid:%"PRIu64" code:%s", *id, tstrerror(terrno)); + continue; + } + + ASSERT(mr.me.type == TSDB_CHILD_TABLE); + if (mr.me.ctbEntry.suid != pScanInfo->tableUid) { + continue; + } + + taosArrayPush(qa, id); + } + + metaReaderClear(&mr); + + qDebug(" %d qualified child tables added into stream scanner", (int32_t) taosArrayGetSize(qa)); + int32_t code = tqReadHandleAddTbUidList(pScanInfo->streamBlockReader, qa); if (code != TSDB_CODE_SUCCESS) { return code; } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 1cfabf2975..c261271728 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -155,7 +155,7 @@ SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, void operatorDummyCloseFn(void* param, int32_t numOfCols) {} static int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, - int32_t orderType, int32_t* rowCellOffset, SqlFunctionCtx* pCtx, int32_t numOfExprs); + int32_t* rowCellOffset, SqlFunctionCtx* pCtx, int32_t numOfExprs); static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size); static void setResultBufSize(STaskAttr* pQueryAttr, SResultInfo* pResultInfo); @@ -2136,23 +2136,13 @@ void setExecutionContext(int32_t numOfOutput, uint64_t groupId, SExecTaskInfo* p * @param result */ int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, - int32_t orderType, int32_t* rowCellOffset, SqlFunctionCtx* pCtx, int32_t numOfExprs) { + int32_t* rowCellOffset, SqlFunctionCtx* pCtx, int32_t numOfExprs) { int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); - int32_t numOfResult = pBlock->info.rows; // there are already exists result rows - - int32_t start = 0; - int32_t step = 1; + int32_t start = pGroupResInfo->index; // qDebug("QInfo:0x%"PRIx64" start to copy data from windowResInfo to output buf", GET_TASKID(pRuntimeEnv)); - assert(orderType == TSDB_ORDER_ASC || orderType == TSDB_ORDER_DESC); - if (orderType == TSDB_ORDER_ASC) { - start = pGroupResInfo->index; - } else { // desc order copy all data - start = numOfRows - pGroupResInfo->index - 1; - } - - for (int32_t i = start; (i < numOfRows) && (i >= 0); i += step) { + for (int32_t i = start; i < numOfRows; i += 1) { SResKeyPos* pPos = taosArrayGetP(pGroupResInfo->pRows, i); SFilePage* page = getBufPage(pBuf, pPos->pos.pageId); @@ -2162,9 +2152,7 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprIn continue; } - // TODO copy multiple rows? - int32_t numOfRowsToCopy = pRow->numOfRows; - if (numOfResult + numOfRowsToCopy >= pBlock->info.capacity) { + if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { break; } @@ -2195,7 +2183,6 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprIn } releaseBufPage(pBuf, page); - pBlock->info.rows += pRow->numOfRows; if (pBlock->info.rows >= pBlock->info.capacity) { // output buffer is full break; @@ -2223,8 +2210,7 @@ void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SG return; } - int32_t orderType = TSDB_ORDER_ASC; - doCopyToSDataBlock(pTaskInfo, pBlock, pExprInfo, pBuf, pGroupResInfo, orderType, rowCellOffset, pCtx, numOfExprs); + doCopyToSDataBlock(pTaskInfo, pBlock, pExprInfo, pBuf, pGroupResInfo, rowCellOffset, pCtx, numOfExprs); // add condition (pBlock->info.rows >= 1) just to runtime happy blockDataUpdateTsWindow(pBlock); @@ -3707,7 +3693,7 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { finalizeMultiTupleQueryResult(pOperator->numOfExprs, pAggInfo->aggSup.pResultBuf, &pAggInfo->binfo.resultRowInfo, pAggInfo->binfo.rowCellInfoOffset); - initGroupedResultInfo(&pAggInfo->groupResInfo, pAggInfo->aggSup.pResultRowHashTable, false); + initGroupedResultInfo(&pAggInfo->groupResInfo, pAggInfo->aggSup.pResultRowHashTable, 0); OPTR_SET_OPENED(pOperator); return TSDB_CODE_SUCCESS; } @@ -4717,10 +4703,11 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SOperatorInfo* pOperatorDumy = createTableScanOperatorInfo(pTableScanNode, pDataReader, pHandle, pTaskInfo); SArray* tableIdList = extractTableIdList(pTableGroupInfo); - SSDataBlock* pResBlock = createResDataBlock(pDescNode); - SArray* pCols = extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID); - SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle->reader, pDataReader, pResBlock, pCols, tableIdList, pTaskInfo, + SSDataBlock* pResBlock = createResDataBlock(pDescNode); + SArray* pCols = extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID); + + SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle->reader, pDataReader, pHandle, pScanPhyNode->uid, pResBlock, pCols, tableIdList, pTaskInfo, pScanPhyNode->node.pConditions, pOperatorDumy); taosArrayDestroy(tableIdList); return pOperator; diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 483ac67e5e..76a773a095 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -314,7 +314,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { // } blockDataEnsureCapacity(pRes, pOperator->resultInfo.capacity); - initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, false); + initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, 0); while(1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 08539206a6..df7478264f 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -102,7 +102,7 @@ static void getNextTimeWindow(SInterval* pInterval, STimeWindow* tw, int32_t ord tw->ekey -= 1; } -static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockInfo) { +static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockInfo, int32_t order) { STimeWindow w = {0}; // 0 by default, which means it is not a interval operator of the upstream operator. @@ -110,13 +110,7 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn return false; } - // todo handle the time range case - TSKEY sk = INT64_MIN; - TSKEY ek = INT64_MAX; - // TSKEY sk = MIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); - // TSKEY ek = MAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); - - if (true) { + if (order == TSDB_ORDER_ASC) { getAlignQueryTimeWindow(pInterval, pInterval->precision, pBlockInfo->window.skey, &w); assert(w.ekey >= pBlockInfo->window.skey); @@ -124,8 +118,8 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn return true; } - while (1) { // todo handle the desc order scan case - getNextTimeWindow(pInterval, &w, TSDB_ORDER_ASC); + while (1) { + getNextTimeWindow(pInterval, &w, order); if (w.skey > pBlockInfo->window.ekey) { break; } @@ -136,24 +130,24 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn } } } else { - // getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.ekey, sk, ek, &w); - // assert(w.skey <= pBlockInfo->window.ekey); - // - // if (w.skey > pBlockInfo->window.skey) { - // return true; - // } - // - // while(1) { - // getNextTimeWindow(pQueryAttr, &w); - // if (w.ekey < pBlockInfo->window.skey) { - // break; - // } - // - // assert(w.skey < pBlockInfo->window.skey); - // if (w.ekey < pBlockInfo->window.ekey && w.ekey >= pBlockInfo->window.skey) { - // return true; - // } - // } + getAlignQueryTimeWindow(pInterval, pInterval->precision, pBlockInfo->window.ekey, &w); + assert(w.skey <= pBlockInfo->window.ekey); + + if (w.skey > pBlockInfo->window.skey) { + return true; + } + + while(1) { + getNextTimeWindow(pInterval, &w, order); + if (w.ekey < pBlockInfo->window.skey) { + break; + } + + assert(w.skey < pBlockInfo->window.skey); + if (w.ekey < pBlockInfo->window.ekey && w.ekey >= pBlockInfo->window.skey) { + return true; + } + } } return false; @@ -172,7 +166,8 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca pCost->totalRows += pBlock->info.rows; *status = pInfo->dataBlockLoadFlag; - if (pTableScanInfo->pFilterNode != NULL || overlapWithTimeWindow(&pTableScanInfo->interval, &pBlock->info)) { + if (pTableScanInfo->pFilterNode != NULL || + overlapWithTimeWindow(&pTableScanInfo->interval, &pBlock->info, pTableScanInfo->cond.order)) { (*status) = FUNC_DATA_REQUIRED_DATA_LOAD; } @@ -188,6 +183,13 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->skipBlocks += 1; + + // clear all data in pBlock that are set when handing the previous block + for(int32_t i = 0; i < pBlockInfo->numOfCols; ++i) { + SColumnInfoData* pcol = taosArrayGet(pBlock->pDataBlock, i); + pcol->pData = NULL; + } + return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_STATIS_LOAD) { pCost->loadBlockStatis += 1; @@ -466,6 +468,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, } pInfo->scanInfo = (SScanInfo){.numOfAsc = pTableScanNode->scanSeq[0], .numOfDesc = pTableScanNode->scanSeq[1]}; +// pInfo->scanInfo = (SScanInfo){.numOfAsc = 0, .numOfDesc = 1}; // for debug purpose pInfo->readHandle = *readHandle; pInfo->interval = extractIntervalInfo(pTableScanNode); @@ -707,14 +710,14 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; blockDataCleanup(pInfo->pRes); - while (tqNextDataBlock(pInfo->readerHandle)) { + while (tqNextDataBlock(pInfo->streamBlockReader)) { SArray* pCols = NULL; uint64_t groupId = 0; uint64_t uid = 0; int32_t numOfRows = 0; int16_t outputCol = 0; - int32_t code = tqRetrieveDataBlock(&pCols, pInfo->readerHandle, &groupId, &uid, &numOfRows, &outputCol); + int32_t code = tqRetrieveDataBlock(&pCols, pInfo->streamBlockReader, &groupId, &uid, &numOfRows, &outputCol); if (code != TSDB_CODE_SUCCESS || numOfRows == 0) { pTaskInfo->code = code; @@ -788,9 +791,10 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { } } -SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, void* pDataReader, - SSDataBlock* pResBlock, SArray* pColList, SArray* pTableIdList, - SExecTaskInfo* pTaskInfo, SNode* pCondition, SOperatorInfo* pOperatorDumy ) { +SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, void* pDataReader, SReadHandle* pHandle, + uint64_t uid, SSDataBlock* pResBlock, SArray* pColList, + SArray* pTableIdList, SExecTaskInfo* pTaskInfo, SNode* pCondition, + SOperatorInfo* pOperatorDumy) { SStreamBlockScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamBlockScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -826,37 +830,32 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, void* pDataR pInfo->tsArray = taosArrayInit(4, sizeof(TSKEY)); if (pInfo->tsArray == NULL) { - taosMemoryFreeClear(pInfo); - taosMemoryFreeClear(pOperator); - return NULL; + goto _error; } pInfo->primaryTsIndex = 0; // TODO(liuyao) get it from physical plan pInfo->pUpdateInfo = updateInfoInitP(&pSTInfo->interval, 10000); // TODO(liuyao) get watermark from physical plan if (pInfo->pUpdateInfo == NULL) { - taosMemoryFreeClear(pInfo); - taosMemoryFreeClear(pOperator); - return NULL; + goto _error; } - pInfo->readerHandle = streamReadHandle; - pInfo->pRes = pResBlock; - pInfo->pCondition = pCondition; - pInfo->pDataReader = pDataReader; - pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; - pInfo->pOperatorDumy = pOperatorDumy; - pInfo->interval = pSTInfo->interval; + pInfo->readHandle = *pHandle; + pInfo->tableUid = uid; + pInfo->streamBlockReader = streamReadHandle; + pInfo->pRes = pResBlock; + pInfo->pCondition = pCondition; + pInfo->pDataReader = pDataReader; + pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; + pInfo->pOperatorDumy = pOperatorDumy; + pInfo->interval = pSTInfo->interval; - pOperator->name = "StreamBlockScanOperator"; + pOperator->name = "StreamBlockScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; pOperator->numOfExprs = pResBlock->info.numOfCols; - pOperator->fpSet._openFn = operatorDummyOpenFn; - pOperator->fpSet.getNextFn = doStreamBlockScan; - pOperator->fpSet.closeFn = operatorDummyCloseFn; - pOperator->pTaskInfo = pTaskInfo; + pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamBlockScan, NULL, NULL, operatorDummyCloseFn, NULL, NULL, NULL); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 79e675e2df..479ce394b1 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -54,8 +54,8 @@ static TSKEY getStartTsKey(STimeWindow* win, const TSKEY* tsCols, int32_t rows, if (tsCols == NULL) { ts = ascQuery ? win->skey : win->ekey; } else { - int32_t offset = ascQuery ? 0 : rows - 1; - ts = tsCols[offset]; +// int32_t offset = ascQuery ? 0 : rows - 1; + ts = tsCols[0]; } return ts; @@ -172,14 +172,22 @@ static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_se } } } else { - int32_t end = searchFn((char*)pData, pos + 1, ekey, order); + int32_t end = searchFn((char*)&pData[pos], numOfRows - pos, ekey, order); if (end >= 0) { - forwardStep = pos - end; + forwardStep = end; - if (pData[end] == ekey) { + if (pData[end + pos] == ekey) { forwardStep += 1; } } +// int32_t end = searchFn((char*)pData, pos + 1, ekey, order); +// if (end >= 0) { +// forwardStep = pos - end; +// +// if (pData[end] == ekey) { +// forwardStep += 1; +// } +// } } assert(forwardStep >= 0); @@ -203,17 +211,25 @@ int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { if (order == TSDB_ORDER_DESC) { // find the first position which is smaller than the key while (1) { - if (key >= keyList[lastPos]) return lastPos; - if (key == keyList[firstPos]) return firstPos; - if (key < keyList[firstPos]) return firstPos - 1; + if (key >= keyList[firstPos]) return firstPos; + if (key == keyList[lastPos]) return lastPos; + + if (key < keyList[lastPos]) { + lastPos += 1; + if (lastPos >= num) { + return -1; + } else { + return lastPos; + } + } numOfRows = lastPos - firstPos + 1; midPos = (numOfRows >> 1) + firstPos; if (key < keyList[midPos]) { - lastPos = midPos - 1; - } else if (key > keyList[midPos]) { firstPos = midPos + 1; + } else if (key > keyList[midPos]) { + lastPos = midPos - 1; } else { break; } @@ -273,12 +289,12 @@ int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimary if (ekey > pDataBlockInfo->window.skey && pPrimaryColumn) { num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); if (item != NULL) { - item->lastKey = pPrimaryColumn[startPos - (num - 1)] + step; + item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step; } } else { - num = startPos + 1; + num = pDataBlockInfo->rows - startPos; if (item != NULL) { - item->lastKey = pDataBlockInfo->window.skey + step; + item->lastKey = pDataBlockInfo->window.ekey + step; } } } @@ -470,20 +486,17 @@ static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, return -1; } - TSKEY startKey = ascQuery ? pNext->skey : pNext->ekey; + TSKEY skey = ascQuery ? pNext->skey : pNext->ekey; int32_t startPos = 0; // tumbling time window query, a special case of sliding time window query if (pInterval->sliding == pInterval->interval && prevPosition != -1) { - int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); - startPos = prevPosition + factor; + startPos = prevPosition + 1; } else { - if (startKey <= pDataBlockInfo->window.skey && ascQuery) { + if ((skey <= pDataBlockInfo->window.skey && ascQuery) || (skey >= pDataBlockInfo->window.ekey && !ascQuery)) { startPos = 0; - } else if (startKey >= pDataBlockInfo->window.ekey && !ascQuery) { - startPos = pDataBlockInfo->rows - 1; } else { - startPos = binarySearchForKey((char*)primaryKeys, pDataBlockInfo->rows, startKey, order); + startPos = binarySearchForKey((char*)primaryKeys, pDataBlockInfo->rows, skey, order); } } @@ -608,7 +621,7 @@ static void saveDataBlockLastRow(char** pRow, SArray* pDataBlock, int32_t rowInd } static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock, - int32_t tableGroupId) { + uint64_t tableGroupId) { SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info; SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; @@ -620,7 +633,7 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe } int32_t step = 1; - bool ascScan = true; + bool ascScan = (pInfo->order == TSDB_ORDER_ASC); // int32_t prevIndex = pResultRowInfo->curPos; @@ -630,7 +643,7 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe tsCols = (int64_t*)pColDataInfo->pData; } - int32_t startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1); + int32_t startPos = 0; TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols, pSDataBlock->info.rows, ascScan); STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, @@ -654,9 +667,10 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe } int32_t forwardStep = 0; - TSKEY ekey = win.ekey; + TSKEY ekey = ascScan? win.ekey:win.skey; forwardStep = - getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); + getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order); + ASSERT(forwardStep > 0); // prev time window not interpolation yet. // int32_t curIndex = pResultRowInfo->curPos; @@ -731,9 +745,9 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe taosArrayPush(pUpdated, &pos); } - ekey = nextWin.ekey; // reviseWindowEkey(pQueryAttr, &nextWin); + ekey = ascScan? nextWin.ekey:nextWin.skey; forwardStep = - getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); + getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order); // window start(end) key interpolation doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardStep, @@ -761,7 +775,8 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SIntervalAggOperatorInfo* pInfo = pOperator->info; - int32_t order = TSDB_ORDER_ASC; + int32_t scanFlag = MAIN_SCAN; + SOperatorInfo* downstream = pOperator->pDownstream[0]; while (1) { @@ -773,8 +788,10 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { break; } + getTableScanInfo(pOperator, &pInfo->order, &scanFlag); + // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order, MAIN_SCAN, true); + setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, pInfo->order, scanFlag, true); STableQueryInfo* pTableQueryInfo = pInfo->pCurrent; setIntervalQueryRange(pTableQueryInfo, pBlock->info.window.skey, &pTaskInfo->window); @@ -800,7 +817,7 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { finalizeMultiTupleQueryResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); - initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); + initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, pInfo->order); OPTR_SET_OPENED(pOperator); return TSDB_CODE_SUCCESS; } @@ -945,7 +962,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { finalizeMultiTupleQueryResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset); - initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); + initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC); blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { @@ -1070,6 +1087,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { doClearWindows(pInfo, pOperator->numOfExprs, pBlock); continue; } + pInfo->order = TSDB_ORDER_ASC; pUpdated = hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, 0); } @@ -1119,7 +1137,6 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pInfo->order = TSDB_ORDER_ASC; pInfo->interval = *pInterval; - // pInfo->execModel = OPTR_EXEC_MODEL_STREAM; pInfo->execModel = pTaskInfo->execModel; pInfo->win = pTaskInfo->window; pInfo->twAggSup = *pTwAggSupp; @@ -1338,7 +1355,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { finalizeMultiTupleQueryResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset); - initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); + initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC); blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index c25d74911c..99313675a5 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -90,6 +90,10 @@ bool histogramFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultIn int32_t histogramFunction(SqlFunctionCtx* pCtx); int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); +bool getHLLFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); +int32_t hllFunction(SqlFunctionCtx* pCtx); +int32_t hllFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); + bool getStateFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool stateFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); int32_t stateCountFunction(SqlFunctionCtx* pCtx); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index e41e3c7c39..48165fdd99 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -263,6 +263,21 @@ static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t l return TSDB_CODE_SUCCESS; } +static int32_t translateHLL(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + if (1 != LIST_LENGTH(pFunc->pParameterList)) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } + + SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); + if (QUERY_NODE_COLUMN != nodeType(pPara)) { + return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, + "The input parameter of HYPERLOGLOG function can only be column"); + } + + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_UBIGINT].bytes, .type = TSDB_DATA_TYPE_UBIGINT}; + return TSDB_CODE_SUCCESS; +} + static int32_t translateStateCount(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { if (3 != LIST_LENGTH(pFunc->pParameterList)) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); @@ -829,6 +844,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .processFunc = histogramFunction, .finalizeFunc = histogramFinalize }, + { + .name = "hyperloglog", + .type = FUNCTION_TYPE_HYPERLOGLOG, + .classification = FUNC_MGT_AGG_FUNC, + .translateFunc = translateHLL, + .getEnvFunc = getHLLFuncEnv, + .initFunc = functionSetup, + .processFunc = hllFunction, + .finalizeFunc = hllFinalize + }, { .name = "state_count", .type = FUNCTION_TYPE_STATE_COUNT, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 479a11ca4a..fd99827c0a 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -28,6 +28,12 @@ #define TAIL_MAX_POINTS_NUM 100 #define TAIL_MAX_OFFSET 100 +#define HLL_BUCKET_BITS 14 // The bits of the bucket +#define HLL_DATA_BITS (64-HLL_BUCKET_BITS) +#define HLL_BUCKETS (1<numOfRes; } +bool getHLLFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { + pEnv->calcMemSize = sizeof(SHLLInfo); + return true; +} + +static uint8_t hllCountNum(void* data, int32_t bytes, int32_t *buk) { + uint64_t hash = MurmurHash3_64(data, bytes); + int32_t index = hash & HLL_BUCKET_MASK; + hash >>= HLL_BUCKET_BITS; + hash |= ((uint64_t)1 << HLL_DATA_BITS); + uint64_t bit = 1; + uint8_t count = 1; + while((hash & bit) == 0) { + count++; + bit <<= 1; + } + *buk = index; + return count; +} + +static void hllBucketHisto(uint8_t *buckets, int32_t* bucketHisto) { + uint64_t *word = (uint64_t*) buckets; + uint8_t *bytes; + + for (int32_t j = 0; j < HLL_BUCKETS>>3; j++) { + if (*word == 0) { + bucketHisto[0] += 8; + } else { + bytes = (uint8_t*) word; + bucketHisto[bytes[0]]++; + bucketHisto[bytes[1]]++; + bucketHisto[bytes[2]]++; + bucketHisto[bytes[3]]++; + bucketHisto[bytes[4]]++; + bucketHisto[bytes[5]]++; + bucketHisto[bytes[6]]++; + bucketHisto[bytes[7]]++; + } + word++; + } +} +static double hllTau(double x) { + if (x == 0. || x == 1.) return 0.; + double zPrime; + double y = 1.0; + double z = 1 - x; + do { + x = sqrt(x); + zPrime = z; + y *= 0.5; + z -= pow(1 - x, 2)*y; + } while(zPrime != z); + return z / 3; +} + +static double hllSigma(double x) { + if (x == 1.0) return INFINITY; + double zPrime; + double y = 1; + double z = x; + do { + x *= x; + zPrime = z; + z += x * y; + y += y; + } while(zPrime != z); + return z; +} + +// estimate the cardinality, the algorithm refer this paper: "New cardinality estimation algorithms for HyperLogLog sketches" +static uint64_t hllCountCnt(uint8_t *buckets) { + double m = HLL_BUCKETS; + int32_t buckethisto[64] = {0}; + hllBucketHisto(buckets,buckethisto); + + double z = m * hllTau((m-buckethisto[HLL_DATA_BITS+1])/(double)m); + for (int j = HLL_DATA_BITS; j >= 1; --j) { + z += buckethisto[j]; + z *= 0.5; + } + z += m * hllSigma(buckethisto[0]/(double)m); + double E = (double)llroundl(HLL_ALPHA_INF*m*m/z); + + return (uint64_t) E; +} + + +int32_t hllFunction(SqlFunctionCtx *pCtx) { + SHLLInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + + SInputColumnInfoData* pInput = &pCtx->input; + SColumnInfoData* pCol = pInput->pData[0]; + + int32_t type = pCol->info.type; + int32_t bytes = pCol->info.bytes; + + int32_t start = pInput->startRowIndex; + int32_t numOfRows = pInput->numOfRows; + + int32_t numOfElems = 0; + for (int32_t i = start; i < numOfRows + start; ++i) { + if (pCol->hasNull && colDataIsNull_s(pCol, i)) { + continue; + } + + numOfElems++; + + char* data = colDataGetData(pCol, i); + if (IS_VAR_DATA_TYPE(type)) { + bytes = varDataLen(data); + data = varDataVal(data); + } + + int32_t index = 0; + uint8_t count = hllCountNum(data, bytes, &index); + uint8_t oldcount = pInfo->buckets[index]; + if (count > oldcount) { + pInfo->buckets[index] = count; + } + + } + + SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1); + return TSDB_CODE_SUCCESS; +} + +int32_t hllFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + SHLLInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + + pInfo->result = hllCountCnt(pInfo->buckets); + + return functionFinalize(pCtx, pBlock); +} + bool getStateFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { pEnv->calcMemSize = sizeof(SStateInfo); return true; @@ -3243,7 +3388,7 @@ int32_t tailFunction(SqlFunctionCtx* pCtx) { if (pInfo->offset >= pInput->numOfRows) { return 0; } else { - pInfo->numOfPoints = MIN(pInfo->numOfPoints, pInput->numOfRows - pInfo->offset); + pInfo->numOfPoints = TMIN(pInfo->numOfPoints, pInput->numOfRows - pInfo->offset); } for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex - pInfo->offset; i += 1) { diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 388ec28b76..192e04c9a3 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -795,7 +795,6 @@ int32_t convertScalarParamToDataBlock(SScalarParam *input, int32_t numOfCols, SS } output->info.hasVarCol = hasVarCol; - //TODO: free the array output->pDataBlock output->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); for (int32_t i = 0; i < numOfCols; ++i) { taosArrayPush(output->pDataBlock, (input + i)->columnData); @@ -809,8 +808,12 @@ int32_t convertDataBlockToScalarParm(SSDataBlock *input, SScalarParam *output) { return -1; } output->numOfRows = input->info.rows; - //TODO: memory - output->columnData = taosArrayGet(input->pDataBlock, 0); + + output->columnData = taosMemoryMalloc(sizeof(SColumnInfoData)); + memcpy(output->columnData, + taosArrayGet(input->pDataBlock, 0), + sizeof(SColumnInfoData)); + return 0; } @@ -833,7 +836,7 @@ int32_t udfcGetUdfTaskResultFromUvTask(SClientUdfTask *task, SClientUvTaskNode * fnDebug("udfc get uv task result. task: %p, uvTask: %p", task, uvTask); if (uvTask->type == UV_TASK_REQ_RSP) { if (uvTask->rspBuf.base != NULL) { - SUdfResponse rsp; + SUdfResponse rsp = {0}; void* buf = decodeUdfResponse(uvTask->rspBuf.base, &rsp); assert(uvTask->rspBuf.len == POINTER_DISTANCE(buf, uvTask->rspBuf.base)); task->errCode = rsp.code; @@ -1284,7 +1287,7 @@ int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle) { task->type = UDF_TASK_SETUP; SUdfSetupRequest *req = &task->_setup.req; - memcpy(req->udfName, udfName, TSDB_FUNC_NAME_LEN); + strncpy(req->udfName, udfName, TSDB_FUNC_NAME_LEN); int32_t errCode = udfcRunUdfUvTask(task, UV_TASK_CONNECT); if (errCode != 0) { @@ -1427,7 +1430,10 @@ int32_t doCallUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t int32_t err = callUdf(handle, callType, &inputBlock, NULL, NULL, &resultBlock, NULL); if (err == 0) { convertDataBlockToScalarParm(&resultBlock, output); + taosArrayDestroy(resultBlock.pDataBlock); } + + taosArrayDestroy(inputBlock.pDataBlock); return err; } @@ -1508,16 +1514,15 @@ int32_t doTeardownUdf(UdfcFuncHandle handle) { udfcRunUdfUvTask(task, UV_TASK_REQ_RSP); - SUdfTeardownResponse *rsp = &task->_teardown.rsp; int32_t err = task->errCode; udfcRunUdfUvTask(task, UV_TASK_DISCONNECT); + fnInfo("tear down udf. udf name: %s, udf func handle: %p", session->udfName, handle); + taosMemoryFree(task->session); taosMemoryFree(task); - fnInfo("tear down udf. udf name: %s, udf func handle: %p", session->udfName, handle); - return err; } @@ -1564,6 +1569,7 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResult } udfRes->interResNum = buf.numOfResult; memcpy(udfRes->interResBuf, buf.buf, buf.bufLen); + freeUdfInterBuf(&buf); return true; } @@ -1621,7 +1627,7 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { blockDataDestroy(inputBlock); taosArrayDestroy(tempBlock.pDataBlock); - taosMemoryFree(newState.buf); + freeUdfInterBuf(&newState); return udfCode; } @@ -1650,6 +1656,8 @@ int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) { GET_RES_INFO(pCtx)->numOfRes = udfRes->finalResNum; } + freeUdfInterBuf(&resultBuf); + int32_t numOfResults = functionFinalizeWithResultBuf(pCtx, pBlock, udfRes->finalResBuf); releaseUdfFuncHandle(pCtx->udfName); return udfCallCode == 0 ? numOfResults : udfCallCode; diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 706bf28be0..19421072f8 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -96,10 +96,14 @@ int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf); int32_t udfdLoadUdf(char *udfName, SUdf *udf) { strcpy(udf->name, udfName); + int32_t err = 0; + err = udfdFillUdfInfoFromMNode(global.clientRpc, udf->name, udf); + if (err != 0) { + fnError("can not retrieve udf from mnode. udf name %s", udfName); + return TSDB_CODE_UDF_LOAD_UDF_FAILURE; + } - udfdFillUdfInfoFromMNode(global.clientRpc, udf->name, udf); - //strcpy(udf->path, "/home/slzhou/TDengine/debug/build/lib/libudf1.so"); - int err = uv_dlopen(udf->path, &udf->lib); + err = uv_dlopen(udf->path, &udf->lib); if (err != 0) { fnError("can not load library %s. error: %s", udf->path, uv_strerror(err)); return TSDB_CODE_UDF_LOAD_UDF_FAILURE; @@ -142,7 +146,7 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) { void udfdProcessSetupRequest(SUvUdfWork* uvUdf, SUdfRequest* request) { // TODO: tracable id from client. connect, setup, call, teardown - fnInfo("%" PRId64 " setup request. udf name: %s", request->seqNum, request->setup.udfName); + fnInfo( "setup request. seq num: %" PRId64 ", udf name: %s", request->seqNum, request->setup.udfName); SUdfSetupRequest *setup = &request->setup; int32_t code = TSDB_CODE_SUCCESS; SUdf *udf = NULL; @@ -222,7 +226,7 @@ void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { SUdfDataBlock input = {0}; convertDataBlockToUdfDataBlock(&call->block, &input); code = udf->scalarProcFunc(&input, &output); - + freeUdfDataDataBlock(&input); convertUdfColumnToDataBlock(&output, &response.callRsp.resultData); freeUdfColumn(&output); break; @@ -242,6 +246,8 @@ void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { .bufLen= udf->bufSize, .numOfResult = 0}; code = udf->aggProcFunc(&input, &call->interBuf, &outBuf); + freeUdfInterBuf(&call->interBuf); + freeUdfDataDataBlock(&input); subRsp->resultBuf = outBuf; break; @@ -251,6 +257,7 @@ void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { .bufLen= udf->bufSize, .numOfResult = 0}; code = udf->aggFinishFunc(&call->interBuf, &outBuf); + freeUdfInterBuf(&call->interBuf); subRsp->resultBuf = outBuf; break; } @@ -270,13 +277,37 @@ void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { encodeUdfResponse(&buf, rsp); uvUdf->output = uv_buf_init(bufBegin, len); + switch (call->callType) { + case TSDB_UDF_CALL_SCALA_PROC: { + tDeleteSSDataBlock(&call->block); + tDeleteSSDataBlock(&subRsp->resultData); + break; + } + case TSDB_UDF_CALL_AGG_INIT: { + freeUdfInterBuf(&subRsp->resultBuf); + break; + } + case TSDB_UDF_CALL_AGG_PROC: { + tDeleteSSDataBlock(&call->block); + freeUdfInterBuf(&subRsp->resultBuf); + break; + } + case TSDB_UDF_CALL_AGG_FIN: { + freeUdfInterBuf(&subRsp->resultBuf); + break; + } + default: + break; + + } + taosMemoryFree(uvUdf->input.base); return; } void udfdProcessTeardownRequest(SUvUdfWork* uvUdf, SUdfRequest* request) { SUdfTeardownRequest *teardown = &request->teardown; - fnInfo("teardown. %" PRId64 "handle:%" PRIx64, request->seqNum, teardown->udfHandle); + fnInfo("teardown. seq number: %" PRId64 ", handle:%" PRIx64, request->seqNum, teardown->udfHandle); SUdfcFuncHandle *handle = (SUdfcFuncHandle *)(teardown->udfHandle); SUdf *udf = handle->udf; bool unloadUdf = false; @@ -344,9 +375,8 @@ void udfdProcessRequest(uv_work_t *req) { void udfdOnWrite(uv_write_t *req, int status) { SUvUdfWork *work = (SUvUdfWork *)req->data; if (status < 0) { - // TODO:log error and process it. + fnError("udfd send response error, length: %zu code: %s", work->output.len, uv_err_name(status)); } - fnDebug("send response. length:%zu, status: %s", work->output.len, uv_err_name(status)); taosMemoryFree(work->output.base); taosMemoryFree(work); taosMemoryFree(req); @@ -545,6 +575,7 @@ void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { taosWriteFile(file, pFuncInfo->pCode, pFuncInfo->codeSize); taosCloseFile(&file); strncpy(udf->path, path, strlen(path)); + tFreeSFuncInfo(pFuncInfo); taosArrayDestroy(retrieveRsp.pFuncInfos); msgInfo->code = 0; } @@ -796,21 +827,26 @@ static int32_t udfdUvInit() { return 0; } +static void udfdCloseWalkCb(uv_handle_t* handle, void* arg) { + if (!uv_is_closing(handle)) { + uv_close(handle, NULL); + } +} + static int32_t udfdRun() { global.udfsHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); uv_mutex_init(&global.udfsMutex); - if (udfdUvInit() != 0) { - fnError("uv init failure"); - return -2; - } + fnInfo("start udfd event loop"); + uv_run(global.loop, UV_RUN_DEFAULT); + fnInfo("udfd event loop stopped."); + + uv_loop_close(global.loop); + + uv_walk(global.loop, udfdCloseWalkCb, NULL); + uv_run(global.loop, UV_RUN_DEFAULT); + uv_loop_close(global.loop); - fnInfo("start the udfd"); - int code = uv_run(global.loop, UV_RUN_DEFAULT); - fnInfo("udfd stopped. result: %s, code: %d", uv_err_name(code), code); - int codeClose = uv_loop_close(global.loop); - fnDebug("uv loop close. result: %s", uv_err_name(codeClose)); - removeListeningPipe(); uv_mutex_destroy(&global.udfsMutex); taosHashCleanup(global.udfsHash); return 0; @@ -853,8 +889,14 @@ int main(int argc, char *argv[]) { return -4; } + if (udfdUvInit() != 0) { + fnError("uv init failure"); + return -5; + } + udfdRun(); - udfdCloseClientRpc(); + removeListeningPipe(); + udfdCloseClientRpc(); } diff --git a/source/libs/function/test/runUdf.c b/source/libs/function/test/runUdf.c index d7c539e5c2..9fe9269a3f 100644 --- a/source/libs/function/test/runUdf.c +++ b/source/libs/function/test/runUdf.c @@ -34,20 +34,13 @@ static int32_t initLog() { return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, NULL, 0); } -int main(int argc, char *argv[]) { - parseArgs(argc, argv); - initLog(); - if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0) != 0) { - fnError("failed to start since read config error"); - return -1; - } - - udfcOpen(); - uv_sleep(1000); - +int scalarFuncTest() { UdfcFuncHandle handle; - doSetupUdf("udf1", &handle); + if (doSetupUdf("udf1", &handle) != 0) { + fnError("setup udf failure"); + return -1; + } SSDataBlock block = {0}; SSDataBlock *pBlock = █ @@ -74,11 +67,78 @@ int main(int argc, char *argv[]) { input.columnData = taosArrayGet(pBlock->pDataBlock, 0); SScalarParam output = {0}; doCallUdfScalarFunc(handle, &input, 1, &output); - + taosArrayDestroy(pBlock->pDataBlock); SColumnInfoData *col = output.columnData; for (int32_t i = 0; i < output.numOfRows; ++i) { fprintf(stderr, "%d\t%d\n", i, *(int32_t *)(col->pData + i * sizeof(int32_t))); } + + colDataDestroy(output.columnData); + taosMemoryFree(output.columnData); + doTeardownUdf(handle); + + return 0; +} + +int aggregateFuncTest() { + UdfcFuncHandle handle; + + if (doSetupUdf("udf2", &handle) != 0) { + fnError("setup udf failure"); + return -1; + } + + SSDataBlock block = {0}; + SSDataBlock *pBlock = █ + pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); + pBlock->info.numOfCols = 1; + pBlock->info.rows = 4; + char data[16] = {0}; + char bitmap[4] = {0}; + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData colInfo = {0}; + colInfo.info.type = TSDB_DATA_TYPE_INT; + colInfo.info.bytes = sizeof(int32_t); + colInfo.info.colId = 1; + colInfo.pData = data; + colInfo.nullbitmap = bitmap; + for (int32_t j = 0; j < pBlock->info.rows; ++j) { + colDataAppendInt32(&colInfo, j, &j); + } + taosArrayPush(pBlock->pDataBlock, &colInfo); + } + + SUdfInterBuf buf = {0}; + SUdfInterBuf newBuf = {0}; + SUdfInterBuf resultBuf = {0}; + doCallUdfAggInit(handle, &buf); + doCallUdfAggProcess(handle, pBlock, &buf, &newBuf); + taosArrayDestroy(pBlock->pDataBlock); + + doCallUdfAggFinalize(handle, &newBuf, &resultBuf); + fprintf(stderr, "agg result: %f\n", *(double*)resultBuf.buf); + + freeUdfInterBuf(&buf); + freeUdfInterBuf(&newBuf); + freeUdfInterBuf(&resultBuf); + doTeardownUdf(handle); + + return 0; +} + +int main(int argc, char *argv[]) { + parseArgs(argc, argv); + initLog(); + if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0) != 0) { + fnError("failed to start since read config error"); + return -1; + } + + udfcOpen(); + uv_sleep(1000); + + scalarFuncTest(); + aggregateFuncTest(); udfcClose(); } diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index eea30bfb03..dd11e135b3 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -266,7 +266,7 @@ int32_t indexConvertData(void* src, int8_t type, void** dst) { TASSERT(0); break; } - *dst = *dst - tlen; + *dst = (char*)*dst - tlen; // indexMayFillNumbericData(*dst, tlen); return tlen; } @@ -306,7 +306,7 @@ int32_t indexConvertDataToStr(void* src, int8_t type, void** dst) { tlen = taosEncodeBinary(NULL, src, sizeof(float)); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeBinary(dst, src, sizeof(float)); - *dst = *dst - tlen; + *dst = (char*) * dst - tlen; break; case TSDB_DATA_TYPE_UINT: *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1); @@ -320,7 +320,7 @@ int32_t indexConvertDataToStr(void* src, int8_t type, void** dst) { tlen = taosEncodeBinary(NULL, src, sizeof(double)); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeBinary(dst, src, sizeof(double)); - *dst = *dst - tlen; + *dst = (char*) * dst - tlen; break; case TSDB_DATA_TYPE_UBIGINT: assert(0); @@ -331,7 +331,7 @@ int32_t indexConvertDataToStr(void* src, int8_t type, void** dst) { tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src)); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeBinary(dst, varDataVal(src), varDataLen(src)); - *dst = *dst - tlen; + *dst = (char*) * dst - tlen; break; } @@ -340,7 +340,7 @@ int32_t indexConvertDataToStr(void* src, int8_t type, void** dst) { tlen = taosEncodeBinary(NULL, src, strlen(src)); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeBinary(dst, src, strlen(src)); - *dst = *dst - tlen; + *dst = (char*) * dst - tlen; break; #endif } @@ -349,7 +349,7 @@ int32_t indexConvertDataToStr(void* src, int8_t type, void** dst) { tlen = taosEncodeBinary(NULL, src, strlen(src)); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeBinary(dst, src, strlen(src)); - *dst = *dst - tlen; + *dst = (char*) * dst - tlen; break; #endif default: diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index c2bde4082c..76c8b17487 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -529,6 +529,18 @@ void nodesDestroyNode(SNodeptr pNode) { nodesDestroyNode(pStmt->pTbNamePattern); break; } + case QUERY_NODE_QUERY: { + SQuery* pQuery = (SQuery*)pNode; + nodesDestroyNode(pQuery->pRoot); + taosMemoryFreeClear(pQuery->pResSchema); + if (NULL != pQuery->pCmdMsg) { + taosMemoryFreeClear(pQuery->pCmdMsg->pMsg); + taosMemoryFreeClear(pQuery->pCmdMsg); + } + taosArrayDestroy(pQuery->pDbList); + taosArrayDestroy(pQuery->pTableList); + break; + } case QUERY_NODE_LOGIC_PLAN_SCAN: { SScanLogicNode* pLogicNode = (SScanLogicNode*)pNode; destroyLogicNode((SLogicNode*)pLogicNode); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 3983b53b6e..df7d10969d 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4254,7 +4254,135 @@ static int32_t rewriteDropTable(STranslateContext* pCxt, SQuery* pQuery) { return rewriteToVnodeModifyOpStmt(pQuery, pBufArray); } -static int32_t buildAlterTbReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, SVAlterTbReq* pReq) { +static SSchema* getColSchema(STableMeta* pTableMeta, const char* pTagName) { + int32_t numOfFields = getNumOfTags(pTableMeta) + getNumOfColumns(pTableMeta); + for (int32_t i = 0; i < numOfFields; ++i) { + SSchema* pTagSchema = pTableMeta->schema + i; + if (0 == strcmp(pTagName, pTagSchema->name)) { + return pTagSchema; + } + } + return NULL; +} + +static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, STableMeta* pTableMeta, + SVAlterTbReq* pReq) { + SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); + if (NULL == pSchema) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE); + } + + pReq->tagName = strdup(pStmt->colName); + if (NULL == pReq->tagName) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + if (DEAL_RES_ERROR == translateValueImpl(pCxt, pStmt->pVal, schemaToDataType(pSchema))) { + return pCxt->errCode; + } + + pReq->isNull = (TSDB_DATA_TYPE_NULL == pStmt->pVal->node.resType.type); + pReq->nTagVal = pStmt->pVal->node.resType.bytes; + char* pVal = nodesGetValueFromNode(pStmt->pVal); + pReq->pTagVal = IS_VAR_DATA_TYPE(pStmt->pVal->node.resType.type) ? pVal + VARSTR_HEADER_SIZE : pVal; + + return TSDB_CODE_SUCCESS; +} + +static int32_t buildAddColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, STableMeta* pTableMeta, + SVAlterTbReq* pReq) { + if (NULL != getColSchema(pTableMeta, pStmt->colName)) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_DUPLICATED_COLUMN); + } + + pReq->colName = strdup(pStmt->colName); + if (NULL == pReq->colName) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pReq->type = pStmt->dataType.type; + pReq->flags = COL_SMA_ON; + pReq->bytes = pStmt->dataType.bytes; + return TSDB_CODE_SUCCESS; +} + +static int32_t buildDropColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, STableMeta* pTableMeta, + SVAlterTbReq* pReq) { + SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); + if (NULL == pSchema) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pStmt->colName); + } else if (PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_CANNOT_DROP_PRIMARY_KEY); + } + + pReq->colName = strdup(pStmt->colName); + if (NULL == pReq->colName) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t buildUpdateColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, STableMeta* pTableMeta, + SVAlterTbReq* pReq) { + pReq->colModBytes = calcTypeBytes(pStmt->dataType); + + SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); + if (NULL == pSchema) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pStmt->colName); + } else if (!IS_VAR_DATA_TYPE(pSchema->type) || pSchema->bytes >= pReq->colModBytes) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_MODIFY_COL); + } + + pReq->colName = strdup(pStmt->colName); + if (NULL == pReq->colName) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t buildRenameColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, STableMeta* pTableMeta, + SVAlterTbReq* pReq) { + if (NULL == getColSchema(pTableMeta, pStmt->colName)) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pStmt->colName); + } + if (NULL != getColSchema(pTableMeta, pStmt->newColName)) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_DUPLICATED_COLUMN); + } + + pReq->colName = strdup(pStmt->colName); + pReq->colNewName = strdup(pStmt->newColName); + if (NULL == pReq->colName || NULL == pReq->colNewName) { + return TSDB_CODE_OUT_OF_MEMORY; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t buildUpdateOptionsReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, SVAlterTbReq* pReq) { + int32_t code = TSDB_CODE_SUCCESS; + + if (-1 != pStmt->pOptions->ttl) { + code = checkRangeOption(pCxt, "ttl", pStmt->pOptions->ttl, TSDB_MIN_TABLE_TTL, INT32_MAX); + if (TSDB_CODE_SUCCESS == code) { + pReq->updateTTL = true; + pReq->newTTL = pStmt->pOptions->ttl; + } + } + + if (TSDB_CODE_SUCCESS == code && '\0' != pStmt->pOptions->comment[0]) { + pReq->updateComment = true; + pReq->newComment = strdup(pStmt->pOptions->comment); + if (NULL == pReq->newComment) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + + return code; +} + +static int32_t buildAlterTbReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, STableMeta* pTableMeta, + SVAlterTbReq* pReq) { pReq->tbName = strdup(pStmt->tableName); if (NULL == pReq->tbName) { return TSDB_CODE_OUT_OF_MEMORY; @@ -4268,60 +4396,22 @@ static int32_t buildAlterTbReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, case TSDB_ALTER_TABLE_UPDATE_TAG_BYTES: return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE); case TSDB_ALTER_TABLE_UPDATE_TAG_VAL: - pReq->tagName = strdup(pStmt->colName); - if (NULL == pReq->tagName) { - return TSDB_CODE_OUT_OF_MEMORY; - } - if (DEAL_RES_ERROR == translateValue(pCxt, pStmt->pVal)) { - return pCxt->errCode; - } - pReq->isNull = (TSDB_DATA_TYPE_NULL == pStmt->pVal->node.resType.type); - pReq->nTagVal = pStmt->pVal->node.resType.bytes; - char* pVal = nodesGetValueFromNode(pStmt->pVal); - pReq->pTagVal = IS_VAR_DATA_TYPE(pStmt->pVal->node.resType.type) ? pVal + VARSTR_HEADER_SIZE : pVal; - break; + return buildUpdateTagValReq(pCxt, pStmt, pTableMeta, pReq); case TSDB_ALTER_TABLE_ADD_COLUMN: + return buildAddColReq(pCxt, pStmt, pTableMeta, pReq); case TSDB_ALTER_TABLE_DROP_COLUMN: - pReq->colName = strdup(pStmt->colName); - if (NULL == pReq->colName) { - return TSDB_CODE_OUT_OF_MEMORY; - } - pReq->type = pStmt->dataType.type; - pReq->flags = COL_SMA_ON; - pReq->bytes = pStmt->dataType.bytes; - break; + return buildDropColReq(pCxt, pStmt, pTableMeta, pReq); case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: - pReq->colName = strdup(pStmt->colName); - if (NULL == pReq->colName) { - return TSDB_CODE_OUT_OF_MEMORY; - } - pReq->colModBytes = calcTypeBytes(pStmt->dataType); - break; + return buildUpdateColReq(pCxt, pStmt, pTableMeta, pReq); case TSDB_ALTER_TABLE_UPDATE_OPTIONS: - if (-1 != pStmt->pOptions->ttl) { - pReq->updateTTL = true; - pReq->newTTL = pStmt->pOptions->ttl; - } - if ('\0' != pStmt->pOptions->comment[0]) { - pReq->updateComment = true; - pReq->newComment = strdup(pStmt->pOptions->comment); - if (NULL == pReq->newComment) { - return TSDB_CODE_OUT_OF_MEMORY; - } - } - break; + return buildUpdateOptionsReq(pCxt, pStmt, pReq); case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: - pReq->colName = strdup(pStmt->colName); - pReq->colNewName = strdup(pStmt->newColName); - if (NULL == pReq->colName || NULL == pReq->colNewName) { - return TSDB_CODE_OUT_OF_MEMORY; - } - break; + return buildRenameColReq(pCxt, pStmt, pTableMeta, pReq); default: break; } - return TSDB_CODE_SUCCESS; + return TSDB_CODE_FAILED; } static int32_t serializeAlterTbReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, SVAlterTbReq* pReq, @@ -4394,7 +4484,7 @@ static int32_t rewriteAlterTable(STranslateContext* pCxt, SQuery* pQuery) { } SVAlterTbReq req = {0}; - code = buildAlterTbReq(pCxt, pStmt, &req); + code = buildAlterTbReq(pCxt, pStmt, pTableMeta, &req); SArray* pArray = NULL; if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index e7716741ed..cb2d1b7c07 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -154,6 +154,10 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "Invalid password"; case TSDB_CODE_PAR_INVALID_ALTER_TABLE: return "Invalid alter table statement"; + case TSDB_CODE_PAR_CANNOT_DROP_PRIMARY_KEY: + return "Primary timestamp column cannot be dropped"; + case TSDB_CODE_PAR_INVALID_MODIFY_COL: + return "Only binary/nchar column length could be modified"; case TSDB_CODE_OUT_OF_MEMORY: return "Out of memory"; default: diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 5962867869..82d1aabd2a 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -39,10 +39,16 @@ static int32_t parseSqlIntoAst(SParseContext* pCxt, SQuery** pQuery) { if (TSDB_CODE_SUCCESS == code) { code = authenticate(pCxt, *pQuery); } - if (TSDB_CODE_SUCCESS == code && 0 == (*pQuery)->placeholderNum) { + + if (TSDB_CODE_SUCCESS == code && (*pQuery)->placeholderNum > 0) { + // TSWAP((*pQuery)->pContainPlaceholderRoot, (*pQuery)->pRoot); + return TSDB_CODE_SUCCESS; + } + + if (TSDB_CODE_SUCCESS == code) { code = translate(pCxt, *pQuery); } - if (TSDB_CODE_SUCCESS == code && 0 == (*pQuery)->placeholderNum) { + if (TSDB_CODE_SUCCESS == code) { code = calculateConstant(pCxt, *pQuery); } return code; @@ -142,26 +148,13 @@ int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery) { return code; } -void qDestroyQuery(SQuery* pQueryNode) { - if (NULL == pQueryNode) { - return; - } - nodesDestroyNode(pQueryNode->pRoot); - taosMemoryFreeClear(pQueryNode->pResSchema); - if (NULL != pQueryNode->pCmdMsg) { - taosMemoryFreeClear(pQueryNode->pCmdMsg->pMsg); - taosMemoryFreeClear(pQueryNode->pCmdMsg); - } - taosArrayDestroy(pQueryNode->pDbList); - taosArrayDestroy(pQueryNode->pTableList); - taosMemoryFreeClear(pQueryNode); -} +void qDestroyQuery(SQuery* pQueryNode) { nodesDestroyNode(pQueryNode); } int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema) { return extractResultSchema(pRoot, numOfCols, pSchema); } -int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx, uint64_t queryId) { +int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx) { int32_t code = TSDB_CODE_SUCCESS; if (colIdx < 0) { @@ -184,6 +177,6 @@ int32_t qStmtParseQuerySql(SParseContext* pCxt, SQuery* pQuery) { if (TSDB_CODE_SUCCESS == code) { code = calculateConstant(pCxt, pQuery); } - + return code; } diff --git a/source/libs/parser/test/parInitialATest.cpp b/source/libs/parser/test/parInitialATest.cpp index e1fa8ffe8d..cc0dded570 100644 --- a/source/libs/parser/test/parInitialATest.cpp +++ b/source/libs/parser/test/parInitialATest.cpp @@ -283,13 +283,13 @@ TEST_F(ParserInitialATest, alterTable) { setAlterColFunc("t1", TSDB_ALTER_TABLE_DROP_COLUMN, "c1"); run("ALTER TABLE t1 DROP COLUMN c1"); - setAlterColFunc("t1", TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, "c1", TSDB_DATA_TYPE_VARCHAR, 20 + VARSTR_HEADER_SIZE); - run("ALTER TABLE t1 MODIFY COLUMN c1 VARCHAR(20)"); + setAlterColFunc("t1", TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, "c2", TSDB_DATA_TYPE_VARCHAR, 30 + VARSTR_HEADER_SIZE); + run("ALTER TABLE t1 MODIFY COLUMN c2 VARCHAR(30)"); setAlterColFunc("t1", TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, "c1", 0, 0, "cc1"); run("ALTER TABLE t1 RENAME COLUMN c1 cc1"); - int64_t val = 10; + int32_t val = 10; setAlterTagFunc("st1s1", "tag1", (const uint8_t*)&val, sizeof(val)); run("ALTER TABLE st1s1 SET TAG tag1=10"); diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index f65e674bf6..8e6c04bb33 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -18,28 +18,6 @@ #include "planInt.h" #include "scalar.h" -typedef struct SCollectPlaceholderValuesCxt { - int32_t errCode; - SArray* pValues; -} SCollectPlaceholderValuesCxt; - -static EDealRes collectPlaceholderValuesImpl(SNode* pNode, void* pContext) { - if (QUERY_NODE_VALUE == nodeType(pNode) && ((SValueNode*)pNode)->placeholderNo > 0) { - SCollectPlaceholderValuesCxt* pCxt = pContext; - taosArrayInsert(pCxt->pValues, ((SValueNode*)pNode)->placeholderNo - 1, &pNode); - return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR; - } - return DEAL_RES_CONTINUE; -} - -static int32_t collectPlaceholderValues(SPlanContext* pCxt, SQueryPlan* pPlan) { - pPlan->pPlaceholderValues = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); - - SCollectPlaceholderValuesCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pValues = pPlan->pPlaceholderValues}; - nodesWalkPhysiPlan((SNode*)pPlan, collectPlaceholderValuesImpl, &cxt); - return cxt.errCode; -} - int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNodeList) { SLogicNode* pLogicNode = NULL; SLogicSubplan* pLogicSubplan = NULL; @@ -58,9 +36,6 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo if (TSDB_CODE_SUCCESS == code) { code = createPhysiPlan(pCxt, pLogicPlan, pPlan, pExecNodeList); } - if (TSDB_CODE_SUCCESS == code && pCxt->placeholderNum > 0) { - code = collectPlaceholderValues(pCxt, *pPlan); - } nodesDestroyNode(pLogicNode); nodesDestroyNode(pLogicSubplan); @@ -99,249 +74,6 @@ int32_t qSetSubplanExecutionNode(SSubplan* subplan, int32_t groupId, SDownstream return setSubplanExecutionNode(subplan->pNode, groupId, pSource); } -static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) { - if (pParam->is_null && 1 == *(pParam->is_null)) { - pVal->node.resType.type = TSDB_DATA_TYPE_NULL; - pVal->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_NULL].bytes; - return TSDB_CODE_SUCCESS; - } - int32_t inputSize = (NULL != pParam->length ? *(pParam->length) : tDataTypes[pParam->buffer_type].bytes); - pVal->node.resType.type = pParam->buffer_type; - pVal->node.resType.bytes = inputSize; - switch (pParam->buffer_type) { - case TSDB_DATA_TYPE_BOOL: - pVal->datum.b = *((bool*)pParam->buffer); - break; - case TSDB_DATA_TYPE_TINYINT: - pVal->datum.i = *((int8_t*)pParam->buffer); - break; - case TSDB_DATA_TYPE_SMALLINT: - pVal->datum.i = *((int16_t*)pParam->buffer); - break; - case TSDB_DATA_TYPE_INT: - pVal->datum.i = *((int32_t*)pParam->buffer); - break; - case TSDB_DATA_TYPE_BIGINT: - pVal->datum.i = *((int64_t*)pParam->buffer); - break; - case TSDB_DATA_TYPE_FLOAT: - pVal->datum.d = *((float*)pParam->buffer); - break; - case TSDB_DATA_TYPE_DOUBLE: - pVal->datum.d = *((double*)pParam->buffer); - break; - case TSDB_DATA_TYPE_VARCHAR: - case TSDB_DATA_TYPE_VARBINARY: - pVal->datum.p = taosMemoryCalloc(1, pVal->node.resType.bytes + VARSTR_HEADER_SIZE + 1); - if (NULL == pVal->datum.p) { - return TSDB_CODE_OUT_OF_MEMORY; - } - varDataSetLen(pVal->datum.p, pVal->node.resType.bytes); - strncpy(varDataVal(pVal->datum.p), (const char*)pParam->buffer, pVal->node.resType.bytes); - break; - case TSDB_DATA_TYPE_NCHAR: { - pVal->node.resType.bytes *= TSDB_NCHAR_SIZE; - pVal->datum.p = taosMemoryCalloc(1, pVal->node.resType.bytes + VARSTR_HEADER_SIZE + 1); - if (NULL == pVal->datum.p) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - int32_t output = 0; - if (!taosMbsToUcs4(pParam->buffer, inputSize, (TdUcs4*)varDataVal(pVal->datum.p), pVal->node.resType.bytes, - &output)) { - return errno; - } - varDataSetLen(pVal->datum.p, output); - pVal->node.resType.bytes = output; - break; - } - case TSDB_DATA_TYPE_TIMESTAMP: - pVal->datum.i = *((int64_t*)pParam->buffer); - break; - case TSDB_DATA_TYPE_UTINYINT: - pVal->datum.u = *((uint8_t*)pParam->buffer); - break; - case TSDB_DATA_TYPE_USMALLINT: - pVal->datum.u = *((uint16_t*)pParam->buffer); - break; - case TSDB_DATA_TYPE_UINT: - pVal->datum.u = *((uint32_t*)pParam->buffer); - break; - case TSDB_DATA_TYPE_UBIGINT: - pVal->datum.u = *((uint64_t*)pParam->buffer); - break; - case TSDB_DATA_TYPE_JSON: - case TSDB_DATA_TYPE_DECIMAL: - case TSDB_DATA_TYPE_BLOB: - case TSDB_DATA_TYPE_MEDIUMBLOB: - // todo - default: - break; - } - pVal->translate = true; - return TSDB_CODE_SUCCESS; -} - -static EDealRes updatePlanQueryId(SNode* pNode, void* pContext) { - int64_t queryId = *(uint64_t*)pContext; - - if (QUERY_NODE_PHYSICAL_PLAN == nodeType(pNode)) { - SQueryPlan* planNode = (SQueryPlan*)pNode; - planNode->queryId = queryId; - } else if (QUERY_NODE_PHYSICAL_SUBPLAN == nodeType(pNode)) { - SSubplan* subplanNode = (SSubplan*)pNode; - subplanNode->id.queryId = queryId; - } - - return DEAL_RES_CONTINUE; -} - -static int32_t calcConstNode(SNode** pNode) { - if (NULL == *pNode) { - return TSDB_CODE_SUCCESS; - } - - SNode* pNew = NULL; - int32_t code = scalarCalculateConstants(*pNode, &pNew); - if (TSDB_CODE_SUCCESS == code) { - *pNode = pNew; - } - return code; -} - -static int32_t calcConstList(SNodeList* pList) { - SNode* pNode = NULL; - FOREACH(pNode, pList) { - SNode* pNew = NULL; - int32_t code = scalarCalculateConstants(pNode, &pNew); - if (TSDB_CODE_SUCCESS == code) { - REPLACE_NODE(pNew); - } else { - return code; - } - } - return TSDB_CODE_SUCCESS; -} - -static bool isEmptyResultCond(SNode** pCond) { - if (NULL == *pCond || QUERY_NODE_VALUE != nodeType(*pCond)) { - return false; - } - if (((SValueNode*)*pCond)->datum.b) { - nodesDestroyNode(*pCond); - *pCond = NULL; - return false; - } - return true; -} - -static int32_t calcConstSpecificPhysiNode(SPhysiNode* pPhyNode) { - switch (nodeType(pPhyNode)) { - case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: - case QUERY_NODE_PHYSICAL_PLAN_FILL: - return TSDB_CODE_SUCCESS; - case QUERY_NODE_PHYSICAL_PLAN_PROJECT: - return calcConstList(((SProjectPhysiNode*)pPhyNode)->pProjections); - case QUERY_NODE_PHYSICAL_PLAN_JOIN: - return calcConstNode(&(((SJoinPhysiNode*)pPhyNode)->pOnConditions)); - case QUERY_NODE_PHYSICAL_PLAN_AGG: - return calcConstList(((SAggPhysiNode*)pPhyNode)->pExprs); - case QUERY_NODE_PHYSICAL_PLAN_SORT: - return calcConstList(((SSortPhysiNode*)pPhyNode)->pExprs); - case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: - case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: - case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW: - return calcConstList(((SWinodwPhysiNode*)pPhyNode)->pExprs); - case QUERY_NODE_PHYSICAL_PLAN_PARTITION: - return calcConstList(((SPartitionPhysiNode*)pPhyNode)->pExprs); - default: - break; - } - return TSDB_CODE_SUCCESS; -} - -static int32_t calcConstSubplan(SPhysiNode* pPhyNode, bool* pEmptyResult) { - int32_t code = calcConstNode(&pPhyNode->pConditions); - if (TSDB_CODE_SUCCESS == code) { - code = calcConstSpecificPhysiNode(pPhyNode); - } - if (TSDB_CODE_SUCCESS != code) { - return code; - } - - *pEmptyResult = isEmptyResultCond(&pPhyNode->pConditions); - if (*pEmptyResult) { - return TSDB_CODE_SUCCESS; - } - - *pEmptyResult = true; - - bool subEmptyResult = false; - SNode* pChild = NULL; - FOREACH(pChild, pPhyNode->pChildren) { - code = calcConstSubplan((SPhysiNode*)pChild, &subEmptyResult); - if (TSDB_CODE_SUCCESS != code) { - return code; - } - if (!subEmptyResult) { - *pEmptyResult = false; - } - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t calcConstPhysiPlan(SQueryPlan* pPlan, bool* pEmptyResult) { - *pEmptyResult = true; - - bool subEmptyResult = false; - SNodeListNode* pNode = nodesListGetNode(pPlan->pSubplans, 0); - SNode* pSubplan = NULL; - FOREACH(pSubplan, pNode->pNodeList) { - int32_t code = calcConstSubplan(((SSubplan*)pSubplan)->pNode, pEmptyResult); - if (TSDB_CODE_SUCCESS != code) { - return code; - } - if (!subEmptyResult) { - *pEmptyResult = false; - } - } - return TSDB_CODE_SUCCESS; -} - -int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_MULTI_BIND* pParams, int32_t colIdx, uint64_t queryId, - bool* pEmptyResult) { - int32_t size = taosArrayGetSize(pPlan->pPlaceholderValues); - int32_t code = 0; - - if (colIdx < 0) { - for (int32_t i = 0; i < size; ++i) { - code = setValueByBindParam((SValueNode*)taosArrayGetP(pPlan->pPlaceholderValues, i), pParams + i); - if (code) { - return code; - } - } - } else { - code = setValueByBindParam((SValueNode*)taosArrayGetP(pPlan->pPlaceholderValues, colIdx), pParams); - if (code) { - return code; - } - } - - if (colIdx < 0 || ((colIdx + 1) == size)) { - nodesWalkPhysiPlan((SNode*)pPlan, updatePlanQueryId, &queryId); - code = calcConstPhysiPlan(pPlan, pEmptyResult); - } - - return code; -} - int32_t qSubPlanToString(const SSubplan* pSubplan, char** pStr, int32_t* pLen) { if (SUBPLAN_TYPE_MODIFY == pSubplan->subplanType) { SDataInserterNode* insert = (SDataInserterNode*)pSubplan->pDataSink; diff --git a/source/libs/planner/test/planStmtTest.cpp b/source/libs/planner/test/planStmtTest.cpp index 0674a69355..6d6eaaf190 100644 --- a/source/libs/planner/test/planStmtTest.cpp +++ b/source/libs/planner/test/planStmtTest.cpp @@ -20,35 +20,49 @@ using namespace std; class PlanStmtTest : public PlannerTestBase { public: - void prepare(const string& sql) { - run(sql); - // todo calloc pBindParams_ - } + void buildParam(TAOS_MULTI_BIND* pBindParams, int32_t index, void* pVal, int32_t type, int32_t bytes = 0) { + TAOS_MULTI_BIND* pBindParam = pBindParams + index; + pBindParam->buffer_type = type; + pBindParam->num = 1; + pBindParam->buffer_length = bytes > 0 ? bytes : tDataTypes[type].bytes; + pBindParam->buffer = taosMemoryCalloc(1, pBindParam->buffer_length); + pBindParam->length = (int32_t*)taosMemoryCalloc(1, sizeof(int32_t)); + pBindParam->is_null = (char*)taosMemoryCalloc(1, sizeof(char)); + *(pBindParam->length) = bytes > 0 ? bytes : tDataTypes[type].bytes; + *(pBindParam->is_null) = 0; - void bindParam(int32_t val) { - TAOS_MULTI_BIND* pBind = pBindParams_ + paramNo_++; - pBind->buffer_type = TSDB_DATA_TYPE_INT; - pBind->num = 1; - pBind->buffer_length = sizeof(int32_t); - pBind->buffer = taosMemoryCalloc(1, pBind->buffer_length); - pBind->length = (int32_t*)taosMemoryCalloc(1, sizeof(int32_t)); - pBind->is_null = (char*)taosMemoryCalloc(1, sizeof(char)); - *((int32_t*)pBind->buffer) = val; - *(pBind->length) = sizeof(int32_t); - *(pBind->is_null) = 0; + switch (type) { + case TSDB_DATA_TYPE_BOOL: + *((bool*)pBindParam->buffer) = *(bool*)pVal; + break; + case TSDB_DATA_TYPE_TINYINT: + *((int8_t*)pBindParam->buffer) = *(int64_t*)pVal; + break; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_TIMESTAMP: + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_BLOB: + case TSDB_DATA_TYPE_MEDIUMBLOB: + default: + break; + } } - - void exec() { - // todo - } - - private: - TAOS_MULTI_BIND* pBindParams_; - int32_t paramNo_; }; TEST_F(PlanStmtTest, stmt) { useDb("root", "test"); - // run("select * from t1 where c1 = ?"); + prepare("SELECT * FROM t1 WHERE c1 = ?"); } diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index 6b038ae8ea..97ff181d7f 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -108,6 +108,57 @@ class PlannerTestBaseImpl { } } + void prepare(const string& sql) { + reset(); + try { + doParseSql(sql, &stmtEnv_.pQuery_, true); + + dump(g_dumpModule); + } catch (...) { + dump(DUMP_MODULE_ALL); + throw; + } + } + + void bindParams(TAOS_MULTI_BIND* pParams, int32_t colIdx) { + try { + doBindParams(stmtEnv_.pQuery_, pParams, colIdx); + + SPlanContext cxt = {0}; + setPlanContext(stmtEnv_.pQuery_, &cxt); + + SLogicNode* pLogicNode = nullptr; + doCreateLogicPlan(&cxt, &pLogicNode); + + doOptimizeLogicPlan(&cxt, pLogicNode); + + SLogicSubplan* pLogicSubplan = nullptr; + doSplitLogicPlan(&cxt, pLogicNode, &pLogicSubplan); + + SQueryLogicPlan* pLogicPlan = nullptr; + doScaleOutLogicPlan(&cxt, pLogicSubplan, &pLogicPlan); + + SQueryPlan* pPlan = nullptr; + doCreatePhysiPlan(&cxt, pLogicPlan, &pPlan); + + dump(g_dumpModule); + } catch (...) { + dump(DUMP_MODULE_ALL); + throw; + } + } + + void exec() { + try { + doParseBoundSql(stmtEnv_.pQuery_); + + dump(g_dumpModule); + } catch (...) { + dump(DUMP_MODULE_ALL); + throw; + } + } + private: struct caseEnv { string acctId_; @@ -117,10 +168,15 @@ class PlannerTestBaseImpl { struct stmtEnv { string sql_; array msgBuf_; + SQuery* pQuery_; + + ~stmtEnv() { qDestroyQuery(pQuery_); } }; struct stmtRes { string ast_; + string prepareAst_; + string boundAst_; string rawLogicPlan_; string optimizedLogicPlan_; string splitLogicPlan_; @@ -132,8 +188,10 @@ class PlannerTestBaseImpl { void reset() { stmtEnv_.sql_.clear(); stmtEnv_.msgBuf_.fill(0); + qDestroyQuery(stmtEnv_.pQuery_); res_.ast_.clear(); + res_.boundAst_.clear(); res_.rawLogicPlan_.clear(); res_.optimizedLogicPlan_.clear(); res_.splitLogicPlan_.clear(); @@ -152,6 +210,9 @@ class PlannerTestBaseImpl { if (DUMP_MODULE_ALL == module || DUMP_MODULE_PARSER == module) { cout << "syntax tree : " << endl; cout << res_.ast_ << endl; + + cout << "bound syntax tree : " << endl; + cout << res_.boundAst_ << endl; } if (DUMP_MODULE_ALL == module || DUMP_MODULE_LOGIC == module) { @@ -187,7 +248,7 @@ class PlannerTestBaseImpl { } } - void doParseSql(const string& sql, SQuery** pQuery) { + void doParseSql(const string& sql, SQuery** pQuery, bool prepare = false) { stmtEnv_.sql_ = sql; transform(stmtEnv_.sql_.begin(), stmtEnv_.sql_.end(), stmtEnv_.sql_.begin(), ::tolower); @@ -200,7 +261,31 @@ class PlannerTestBaseImpl { cxt.msgLen = stmtEnv_.msgBuf_.max_size(); DO_WITH_THROW(qParseSql, &cxt, pQuery); - res_.ast_ = toString((*pQuery)->pRoot); + if (prepare) { + res_.prepareAst_ = toString((*pQuery)->pRoot); + } else { + res_.ast_ = toString((*pQuery)->pRoot); + } + } + + void doBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx) { + DO_WITH_THROW(qStmtBindParams, pQuery, pParams, colIdx); + if (colIdx < 0 || pQuery->placeholderNum == colIdx + 1) { + res_.boundAst_ = toString(pQuery->pRoot); + } + } + + void doParseBoundSql(SQuery* pQuery) { + SParseContext cxt = {0}; + cxt.acctId = atoi(caseEnv_.acctId_.c_str()); + cxt.db = caseEnv_.db_.c_str(); + cxt.pSql = stmtEnv_.sql_.c_str(); + cxt.sqlLen = stmtEnv_.sql_.length(); + cxt.pMsg = stmtEnv_.msgBuf_.data(); + cxt.msgLen = stmtEnv_.msgBuf_.max_size(); + + DO_WITH_THROW(qStmtParseQuerySql, &cxt, pQuery); + res_.ast_ = toString(pQuery->pRoot); } void doCreateLogicPlan(SPlanContext* pCxt, SLogicNode** pLogicNode) { @@ -275,3 +360,11 @@ PlannerTestBase::~PlannerTestBase() {} void PlannerTestBase::useDb(const std::string& acctId, const std::string& db) { impl_->useDb(acctId, db); } void PlannerTestBase::run(const std::string& sql) { return impl_->run(sql); } + +void PlannerTestBase::prepare(const std::string& sql) { return impl_->prepare(sql); } + +void PlannerTestBase::bindParams(TAOS_MULTI_BIND* pParams, int32_t colIdx) { + return impl_->bindParams(pParams, colIdx); +} + +void PlannerTestBase::exec() { return impl_->exec(); } diff --git a/source/libs/planner/test/planTestUtil.h b/source/libs/planner/test/planTestUtil.h index a63bba1a97..5a2050a45e 100644 --- a/source/libs/planner/test/planTestUtil.h +++ b/source/libs/planner/test/planTestUtil.h @@ -19,6 +19,7 @@ #include class PlannerTestBaseImpl; +struct TAOS_MULTI_BIND; class PlannerTestBase : public testing::Test { public: @@ -27,6 +28,10 @@ class PlannerTestBase : public testing::Test { void useDb(const std::string& acctId, const std::string& db); void run(const std::string& sql); + // stmt mode APIs + void prepare(const std::string& sql); + void bindParams(TAOS_MULTI_BIND* pParams, int32_t colIdx); + void exec(); private: std::unique_ptr impl_; diff --git a/source/libs/scalar/test/scalar/CMakeLists.txt b/source/libs/scalar/test/scalar/CMakeLists.txt index 03418d17ac..866297b017 100644 --- a/source/libs/scalar/test/scalar/CMakeLists.txt +++ b/source/libs/scalar/test/scalar/CMakeLists.txt @@ -17,3 +17,7 @@ TARGET_INCLUDE_DIRECTORIES( PUBLIC "${TD_SOURCE_DIR}/source/libs/parser/inc" PRIVATE "${TD_SOURCE_DIR}/source/libs/scalar/inc" ) +#add_test( +# NAME scalarTest +# COMMAND scalarTest +#) diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 5637539fea..4fcdb35e0d 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -1148,6 +1148,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch if (NULL == msg) { SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } + SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); break; } case TDMT_VND_SUBMIT_RSP: { diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index 8574e071f2..84b8688ade 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -28,9 +28,9 @@ struct SPCache { SPage lru; }; -static inline int tdbPCachePageHash(const SPgid *pPgid) { - u32 *t = (u32 *)((pPgid)->fileid); - return t[0] + t[1] + t[2] + t[3] + t[4] + t[5] + (pPgid)->pgno; +static inline uint32_t tdbPCachePageHash(const SPgid *pPgid) { + uint32_t *t = (uint32_t *)((pPgid)->fileid); + return (uint32_t)(t[0] + t[1] + t[2] + t[3] + t[4] + t[5] + (pPgid)->pgno); } #define PAGE_IS_PINNED(pPage) ((pPage)->pLruNext == NULL) diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index fbd5cb3aac..dc36c76027 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -72,7 +72,7 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) { return -1; } - ret = tdbGnrtFileID(pPager->dbFileName, pPager->fid, false); + ret = tdbGnrtFileID(pPager->fd, pPager->fid, false); if (ret < 0) { return -1; } diff --git a/source/libs/tdb/src/db/tdbUtil.c b/source/libs/tdb/src/db/tdbUtil.c index 3746d9358f..4acb83c8e4 100644 --- a/source/libs/tdb/src/db/tdbUtil.c +++ b/source/libs/tdb/src/db/tdbUtil.c @@ -35,10 +35,10 @@ void tdbFree(void *p) { } } -int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique) { +int tdbGnrtFileID(tdb_fd_t fd, uint8_t *fileid, bool unique) { int64_t stDev = 0, stIno = 0; - if (taosDevInoFile(fname, &stDev, &stIno) < 0) { + if (taosDevInoFile(fd, &stDev, &stIno) < 0) { return -1; } diff --git a/source/libs/tdb/src/inc/tdbUtil.h b/source/libs/tdb/src/inc/tdbUtil.h index 29a505fa78..c518e8efcc 100644 --- a/source/libs/tdb/src/inc/tdbUtil.h +++ b/source/libs/tdb/src/inc/tdbUtil.h @@ -28,7 +28,7 @@ extern "C" { #define TDB_ROUND8(x) (((x) + 7) & ~7) -int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique); +int tdbGnrtFileID(tdb_fd_t fd, uint8_t *fileid, bool unique); int tdbGetFileSize(tdb_fd_t fd, int szPage, SPgno *size); void *tdbRealloc(void *ptr, size_t size); diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 3cd05b65cd..aa64e65638 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -110,7 +110,7 @@ void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, cha int64_t taosCopyFile(const char *from, const char *to) { #ifdef WINDOWS assert(0); - return 0; + return -1; #else char buffer[4096]; int64_t size = 0; @@ -190,15 +190,35 @@ int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) { return 0; } -int32_t taosDevInoFile(const char *path, int64_t *stDev, int64_t *stIno) { +int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) { + if (pFile == NULL) { + return 0; + } + assert(pFile->fd >= 0); // Please check if you have closed the file. - struct stat fileStat; #ifdef WINDOWS - int32_t code = _stat(path, &fileStat); + + BY_HANDLE_FILE_INFORMATION bhfi; + HANDLE handle = (HANDLE)_get_osfhandle(pFile->fd); + if (GetFileInformationByHandle(handle, &bhfi) == FALSE) { + printf("taosFStatFile get file info fail."); + return -1; + } + + if (stDev != NULL) { + *stDev = (int64_t)(bhfi.dwVolumeSerialNumber); + } + + if (stIno != NULL) { + *stIno = (int64_t)((((uint64_t)bhfi.nFileIndexHigh) << 32) + bhfi.nFileIndexLow); + } + #else - int32_t code = stat(path, &fileStat); -#endif + + struct stat fileStat; + int32_t code = fstat(pFile->fd, &fileStat); if (code < 0) { + printf("taosFStatFile run fstat fail."); return code; } @@ -209,6 +229,7 @@ int32_t taosDevInoFile(const char *path, int64_t *stDev, int64_t *stIno) { if (stIno != NULL) { *stIno = fileStat.st_ino; } +#endif return 0; } diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index fd6172e04f..4d7b15401c 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -744,7 +744,8 @@ int32_t taosGetSystemUUID(char *uid, int32_t uidlen) { #ifdef WINDOWS GUID guid; CoCreateGuid(&guid); - memcpy(uid, &guid, uidlen); + snprintf(uid, uidlen, "%08X-%04X-%04X-%02X%02X-%02X%02X%02X%02X%02X%02X", guid.Data1, guid.Data2, guid.Data3, guid.Data4[0], + guid.Data4[1], guid.Data4[2], guid.Data4[3], guid.Data4[4], guid.Data4[5], guid.Data4[6], guid.Data4[7]); return 0; #elif defined(_TD_DARWIN_64) diff --git a/source/util/src/thashutil.c b/source/util/src/thashutil.c index d5182cb892..c2382550a6 100644 --- a/source/util/src/thashutil.c +++ b/source/util/src/thashutil.c @@ -30,7 +30,7 @@ (h) ^= (h) >> 13; \ (h) *= 0xc2b2ae35; \ (h) ^= (h) >> 16; } while (0) - + uint32_t MurmurHash3_32(const char *key, uint32_t len) { const uint8_t *data = (const uint8_t *)key; const int32_t nblocks = len >> 2u; @@ -78,18 +78,54 @@ uint32_t MurmurHash3_32(const char *key, uint32_t len) { return h1; } +uint64_t MurmurHash3_64(const char *key, uint32_t len) { + const uint64_t m = 0x87c37b91114253d5; + const int r = 47; + uint32_t seed = 0x12345678; + uint64_t h = seed ^ (len * m); + const uint8_t *data = (const uint8_t *)key; + const uint8_t *end = data + (len-(len&7)); + + while(data != end) { + uint64_t k = *((uint64_t*)data); + + k *= m; + k ^= k >> r; + k *= m; + h ^= k; + h *= m; + data += 8; + } + + switch(len & 7) { + case 7: h ^= (uint64_t)data[6] << 48; /* fall-thru */ + case 6: h ^= (uint64_t)data[5] << 40; /* fall-thru */ + case 5: h ^= (uint64_t)data[4] << 32; /* fall-thru */ + case 4: h ^= (uint64_t)data[3] << 24; /* fall-thru */ + case 3: h ^= (uint64_t)data[2] << 16; /* fall-thru */ + case 2: h ^= (uint64_t)data[1] << 8; /* fall-thru */ + case 1: h ^= (uint64_t)data[0]; + h *= m; /* fall-thru */ + }; + + h ^= h >> r; + h *= m; + h ^= h >> r; + return h; +} + uint32_t taosIntHash_32(const char *key, uint32_t UNUSED_PARAM(len)) { return *(uint32_t *)key; } uint32_t taosIntHash_16(const char *key, uint32_t UNUSED_PARAM(len)) { return *(uint16_t *)key; } uint32_t taosIntHash_8(const char *key, uint32_t UNUSED_PARAM(len)) { return *(uint8_t *)key; } uint32_t taosFloatHash(const char *key, uint32_t UNUSED_PARAM(len)) { - float f = GET_FLOAT_VAL(key); + float f = GET_FLOAT_VAL(key); if (isnan(f)) { return 0x7fc00000; } - + if (FLT_EQUAL(f, 0.0)) { return 0; - } + } if (fabs(f) < FLT_MAX/BASE - DLT) { int32_t t = (int32_t)(round(BASE * (f + DLT))); return (uint32_t)t; @@ -98,27 +134,27 @@ uint32_t taosFloatHash(const char *key, uint32_t UNUSED_PARAM(len)) { } } uint32_t taosDoubleHash(const char *key, uint32_t UNUSED_PARAM(len)) { - double f = GET_DOUBLE_VAL(key); + double f = GET_DOUBLE_VAL(key); if (isnan(f)) { return 0x7fc00000; } if (FLT_EQUAL(f, 0.0)) { return 0; - } + } if (fabs(f) < DBL_MAX/BASE - DLT) { int32_t t = (int32_t)(round(BASE * (f + DLT))); return (uint32_t)t; } else { return 0x7fc00000; - } + } } uint32_t taosIntHash_64(const char *key, uint32_t UNUSED_PARAM(len)) { uint64_t val = *(uint64_t *)key; uint64_t hash = val >> 16U; hash += (val & 0xFFFFU); - + return (uint32_t)hash; } @@ -127,39 +163,39 @@ _hash_fn_t taosGetDefaultHashFunction(int32_t type) { switch(type) { case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_UBIGINT: - case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_BIGINT: fn = taosIntHash_64; break; - case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_BINARY: fn = MurmurHash3_32; break; - case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_NCHAR: fn = MurmurHash3_32; break; case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_INT: - fn = taosIntHash_32; + case TSDB_DATA_TYPE_INT: + fn = taosIntHash_32; break; - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: - fn = taosIntHash_16; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + fn = taosIntHash_16; break; case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_TINYINT: - fn = taosIntHash_8; + case TSDB_DATA_TYPE_TINYINT: + fn = taosIntHash_8; break; - case TSDB_DATA_TYPE_FLOAT: - fn = taosFloatHash; - break; - case TSDB_DATA_TYPE_DOUBLE: - fn = taosDoubleHash; - break; - default: + case TSDB_DATA_TYPE_FLOAT: + fn = taosFloatHash; + break; + case TSDB_DATA_TYPE_DOUBLE: + fn = taosDoubleHash; + break; + default: fn = taosIntHash_32; break; } - + return fn; } diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index 558c2258f4..c5cd968fd9 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -183,8 +183,12 @@ int32_t tjsonGetBigIntValue(const SJson* pJson, const char* pName, int64_t* pVal if (NULL == p) { return TSDB_CODE_FAILED; } - +#ifdef WINDOWS + sscanf(p,"%lld",pVal); +#else + // sscanf(p,"%ld",pVal); *pVal = strtol(p, NULL, 10); +#endif return TSDB_CODE_SUCCESS; } @@ -214,8 +218,12 @@ int32_t tjsonGetUBigIntValue(const SJson* pJson, const char* pName, uint64_t* pV if (NULL == p) { return TSDB_CODE_FAILED; } - +#ifdef WINDOWS + sscanf(p,"%llu",pVal); +#else + // sscanf(p,"%ld",pVal); *pVal = strtoul(p, NULL, 10); +#endif return TSDB_CODE_SUCCESS; } diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index 17a87b1a5f..5e206f3e6e 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -33,11 +33,11 @@ typedef struct STaosQnode { } STaosQnode; typedef struct STaosQueue { - STaosQnode *head; - STaosQnode *tail; - STaosQueue *next; // for queue set - STaosQset *qset; // for queue set - void *ahandle; // for queue set + STaosQnode * head; + STaosQnode * tail; + STaosQueue * next; // for queue set + STaosQset * qset; // for queue set + void * ahandle; // for queue set FItem itemFp; FItems itemsFp; TdThreadMutex mutex; @@ -46,8 +46,8 @@ typedef struct STaosQueue { } STaosQueue; typedef struct STaosQset { - STaosQueue *head; - STaosQueue *current; + STaosQueue * head; + STaosQueue * current; TdThreadMutex mutex; tsem_t sem; int32_t numOfQueues; @@ -85,7 +85,7 @@ void taosSetQueueFp(STaosQueue *queue, FItem itemFp, FItems itemsFp) { void taosCloseQueue(STaosQueue *queue) { if (queue == NULL) return; STaosQnode *pTemp; - STaosQset *qset; + STaosQset * qset; taosThreadMutexLock(&queue->mutex); STaosQnode *pNode = queue->head; @@ -152,7 +152,7 @@ void *taosAllocateQitem(int32_t size, EQItype itype) { if (itype == RPC_QITEM) { int64_t alloced = atomic_add_fetch_64(&tsRpcQueueMemoryUsed, size); - if (alloced > tsRpcQueueMemoryUsed) { + if (alloced > tsRpcQueueMemoryAllowed) { taosMemoryFree(pNode); terrno = TSDB_CODE_OUT_OF_RPC_MEMORY_QUEUE; return NULL; diff --git a/source/util/test/procTest.cpp b/source/util/test/procTest.cpp index 7ffec04a40..d77180138c 100644 --- a/source/util/test/procTest.cpp +++ b/source/util/test/procTest.cpp @@ -16,11 +16,11 @@ typedef struct STestMsg { uint16_t msgType; - void *pCont; + void * pCont; int contLen; int32_t code; - void *handle; // rpc handle returned to app - void *ahandle; // app handle set by client + void * handle; // rpc handle returned to app + void * ahandle; // app handle set by client int noResp; // has response or not(default 0, 0: resp, 1: no resp); int persistHandle; // persist handle or not } STestMsg; @@ -37,7 +37,7 @@ class UtilTesProc : public ::testing::Test { head.msgType = 2; head.noResp = 3; head.persistHandle = 4; - + tsRpcQueueMemoryAllowed = 1024 * 1024 * 64; taosRemoveDir("/tmp/td"); taosMkDir("/tmp/td"); tstrncpy(tsLogDir, "/tmp/td", PATH_MAX); @@ -64,18 +64,18 @@ TEST_F(UtilTesProc, 00_Init_Cleanup) { shm.size = 1023; SProcCfg cfg = {(ProcConsumeFp)NULL, - (ProcMallocFp)taosAllocateQitem, - (ProcFreeFp)taosFreeQitem, - (ProcMallocFp)taosMemoryMalloc, - (ProcFreeFp)taosMemoryMalloc, - (ProcConsumeFp)NULL, - (ProcMallocFp)taosMemoryMalloc, - (ProcFreeFp)taosMemoryFree, - (ProcMallocFp)taosMemoryMalloc, - (ProcFreeFp)taosMemoryMalloc, - shm, - &shm, - "1234"}; + (ProcMallocFp)taosAllocateQitem, + (ProcFreeFp)taosFreeQitem, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryMalloc, + (ProcConsumeFp)NULL, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryMalloc, + shm, + &shm, + "1234"}; SProcObj *proc = taosProcInit(&cfg); ASSERT_EQ(proc, nullptr); @@ -105,18 +105,18 @@ TEST_F(UtilTesProc, 01_Push_Pop_Child) { shm.size = 3000; ASSERT_EQ(taosCreateShm(&shm, 1235, shm.size), 0); SProcCfg cfg = {(ProcConsumeFp)ConsumeChild1, - (ProcMallocFp)taosAllocateQitem, - (ProcFreeFp)taosFreeQitem, - (ProcMallocFp)taosMemoryMalloc, - (ProcFreeFp)taosMemoryFree, - (ProcConsumeFp)NULL, - (ProcMallocFp)taosMemoryMalloc, - (ProcFreeFp)taosMemoryFree, - (ProcMallocFp)taosMemoryMalloc, - (ProcFreeFp)taosMemoryFree, - shm, - (void *)((int64_t)1235), - "1235_c"}; + (ProcMallocFp)taosAllocateQitem, + (ProcFreeFp)taosFreeQitem, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + (ProcConsumeFp)NULL, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + shm, + (void *)((int64_t)1235), + "1235_c"}; SProcObj *cproc = taosProcInit(&cfg); ASSERT_NE(cproc, nullptr); @@ -163,18 +163,18 @@ TEST_F(UtilTesProc, 02_Push_Pop_Parent) { shm.size = 3000; ASSERT_EQ(taosCreateShm(&shm, 1236, shm.size), 0); SProcCfg cfg = {(ProcConsumeFp)NULL, - (ProcMallocFp)taosAllocateQitem, - (ProcFreeFp)taosFreeQitem, - (ProcMallocFp)taosMemoryMalloc, - (ProcFreeFp)taosMemoryFree, - (ProcConsumeFp)ConsumeParent1, - (ProcMallocFp)taosMemoryMalloc, - (ProcFreeFp)taosMemoryFree, - (ProcMallocFp)taosMemoryMalloc, - (ProcFreeFp)taosMemoryFree, - shm, - (void *)((int64_t)1236), - "1236_c"}; + (ProcMallocFp)taosAllocateQitem, + (ProcFreeFp)taosFreeQitem, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + (ProcConsumeFp)ConsumeParent1, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + shm, + (void *)((int64_t)1236), + "1236_c"}; SProcObj *cproc = taosProcInit(&cfg); ASSERT_NE(cproc, nullptr); @@ -217,18 +217,18 @@ TEST_F(UtilTesProc, 03_Handle) { shm.size = 3000; ASSERT_EQ(taosCreateShm(&shm, 1237, shm.size), 0); SProcCfg cfg = {(ProcConsumeFp)ConsumeChild3, - (ProcMallocFp)taosAllocateQitem, - (ProcFreeFp)taosFreeQitem, - (ProcMallocFp)taosMemoryMalloc, - (ProcFreeFp)taosMemoryFree, - (ProcConsumeFp)NULL, - (ProcMallocFp)taosMemoryMalloc, - (ProcFreeFp)taosMemoryFree, - (ProcMallocFp)taosMemoryMalloc, - (ProcFreeFp)taosMemoryFree, - shm, - (void *)((int64_t)1235), - "1237_p"}; + (ProcMallocFp)taosAllocateQitem, + (ProcFreeFp)taosFreeQitem, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + (ProcConsumeFp)NULL, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + shm, + (void *)((int64_t)1235), + "1237_p"}; SProcObj *cproc = taosProcInit(&cfg); ASSERT_NE(cproc, nullptr); @@ -236,7 +236,8 @@ TEST_F(UtilTesProc, 03_Handle) { int32_t i = 0; for (i = 0; i < 20; ++i) { head.handle = (void *)((int64_t)i); - ASSERT_EQ(taosProcPutToChildQ(cproc, &head, sizeof(STestMsg), body, i, (void *)((int64_t)i), i, PROC_FUNC_REQ), 0); + ASSERT_EQ(taosProcPutToChildQ(cproc, &head, sizeof(STestMsg), body, i, (void *)((int64_t)i), i, PROC_FUNC_REQ), + 0); } cfg.isChild = true; @@ -247,7 +248,7 @@ TEST_F(UtilTesProc, 03_Handle) { taosProcCleanup(pproc); int64_t ref = 0; - + ref = taosProcRemoveHandle(cproc, (void *)((int64_t)3)); EXPECT_EQ(ref, 3); ref = taosProcRemoveHandle(cproc, (void *)((int64_t)5));