From 32577d41966059b7a66ba3ce5a00e5c2a270319c Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 14 Apr 2022 18:24:21 +0800 Subject: [PATCH 01/47] refactor: add json tag function for sql parser --- include/common/tdataformat.h | 3 +- include/common/ttypes.h | 2 +- include/util/taoserror.h | 1 + include/util/tdef.h | 8 ++ include/util/tjson.h | 1 + include/util/tutil.h | 2 - source/common/src/ttypes.c | 3 +- source/common/src/tvariant.c | 2 +- source/libs/parser/inc/parUtil.h | 1 + source/libs/parser/src/parAstCreater.c | 1 + source/libs/parser/src/parInsert.c | 54 ++++---- source/libs/parser/src/parTranslater.c | 24 +++- source/libs/parser/src/parUtil.c | 169 ++++++++++++++++++++++++- source/libs/scalar/src/filter.c | 2 +- source/util/src/tjson.c | 1 + source/util/src/tutil.c | 59 --------- tools/shell/src/shellEngine.c | 71 +++-------- 17 files changed, 252 insertions(+), 152 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 8a87a1bd2e..2dd887ae74 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -608,7 +608,7 @@ void tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder); void tdResetKVRowBuilder(SKVRowBuilder *pBuilder); SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder); -static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t colId, int8_t type, const void *value) { +static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t colId, const void *value, int32_t tlen) { if (pBuilder->nCols >= pBuilder->tCols) { pBuilder->tCols *= 2; SColIdx *pColIdx = (SColIdx *)taosMemoryRealloc((void *)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols); @@ -621,7 +621,6 @@ static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t co pBuilder->nCols++; - int32_t tlen = IS_VAR_DATA_TYPE(type) ? varDataTLen(value) : TYPE_BYTES[type]; if (tlen > pBuilder->alloc - pBuilder->size) { while (tlen > pBuilder->alloc - pBuilder->size) { pBuilder->alloc *= 2; diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 032cb44122..1112021124 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -252,7 +252,7 @@ typedef struct tDataTypeDescriptor { int16_t *maxindex, int16_t *numofnull); } tDataTypeDescriptor; -extern tDataTypeDescriptor tDataTypes[15]; +extern tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX]; bool isValidDataType(int32_t type); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 60535e2f49..a11bb5162d 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -599,6 +599,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_GROUPBY_WINDOW_COEXIST TAOS_DEF_ERROR_CODE(0, 0x2624) #define TSDB_CODE_PAR_INVALID_OPTION_UNIT TAOS_DEF_ERROR_CODE(0, 0x2625) #define TSDB_CODE_PAR_INVALID_KEEP_UNIT TAOS_DEF_ERROR_CODE(0, 0x2626) +#define TSDB_CODE_PAR_ONLY_ONE_JSON_TAG TAOS_DEF_ERROR_CODE(0, 0x2627) //planner #define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700) diff --git a/include/util/tdef.h b/include/util/tdef.h index f276a1a812..0289bb0df7 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -254,6 +254,14 @@ typedef enum ELogicConditionType { #define TSDB_MAX_TAGS 128 #define TSDB_MAX_TAG_CONDITIONS 1024 +#define TSDB_MAX_JSON_TAG_LEN 16384 +#define TSDB_JSON_PLACEHOLDER 0x7F +#define TSDB_JSON_null 0x00 +#define TSDB_JSON_KEY_NULL 0x00 +#define TSDB_JSON_NOT_NULL 0x01 +#define TSDB_JSON_NULL 0x00 + + #define TSDB_AUTH_LEN 16 #define TSDB_PASSWORD_LEN 32 #define TSDB_USET_PASSWORD_LEN 129 diff --git a/include/util/tjson.h b/include/util/tjson.h index 6b2221f704..9f347e5896 100644 --- a/include/util/tjson.h +++ b/include/util/tjson.h @@ -76,6 +76,7 @@ char* tjsonToString(const SJson* pJson); char* tjsonToUnformattedString(const SJson* pJson); SJson* tjsonParse(const char* pStr); +bool tjsonIsObject(const SJson* pJson); #ifdef __cplusplus } diff --git a/include/util/tutil.h b/include/util/tutil.h index e0f92be76a..444a893a88 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -26,8 +26,6 @@ extern "C" { #endif int32_t strdequote(char *src); -int32_t strndequote(char *dst, const char *z, int32_t len); -int32_t strRmquote(char *z, int32_t len); size_t strtrim(char *src); char *strnchr(const char *haystack, char needle, int32_t len, bool skipquote); char **strsplit(char *src, const char *delim, int32_t *num); diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index ef75adeb5d..6af4777537 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -365,7 +365,7 @@ static void getStatics_nchr(const void *pData, int32_t numOfRow, int64_t *min, i *maxIndex = 0; } -tDataTypeDescriptor tDataTypes[15] = { +tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX] = { {TSDB_DATA_TYPE_NULL, 6, 1, "NOTYPE", 0, 0, NULL, NULL, NULL}, {TSDB_DATA_TYPE_BOOL, 4, CHAR_BYTES, "BOOL", false, true, tsCompressBool, tsDecompressBool, getStatics_bool}, {TSDB_DATA_TYPE_TINYINT, 7, CHAR_BYTES, "TINYINT", INT8_MIN, INT8_MAX, tsCompressTinyint, tsDecompressTinyint, @@ -388,6 +388,7 @@ tDataTypeDescriptor tDataTypes[15] = { {TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", 0, UINT32_MAX, tsCompressInt, tsDecompressInt, getStatics_u32}, {TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", 0, UINT64_MAX, tsCompressBigint, tsDecompressBigint, getStatics_u64}, + {TSDB_DATA_TYPE_JSON, 4, TSDB_MAX_JSON_TAG_LEN, "JSON", 0, 0, tsCompressString, tsDecompressString, getStatics_nchr}, }; char tTokenTypeSwitcher[13] = { diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index a0503e43a1..a546d9bb6b 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -118,7 +118,7 @@ void taosVariantCreate(SVariant *pVar, const char *z, int32_t n, int32_t type) { } case TSDB_DATA_TYPE_BINARY: { pVar->pz = strndup(z, n); - pVar->nLen = strRmquote(pVar->pz, n); + //pVar->nLen = strRmquote(pVar->pz, n); break; } case TSDB_DATA_TYPE_TIMESTAMP: { diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 742ab303d3..460a089802 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -47,6 +47,7 @@ SSchema *getTableTagSchema(const STableMeta* pTableMeta); int32_t getNumOfColumns(const STableMeta* pTableMeta); int32_t getNumOfTags(const STableMeta* pTableMeta); STableComInfo getTableInfo(const STableMeta* pTableMeta); +int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* errMsg, int16_t startColId); int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen); diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index a14053ada7..0dcf04ecc0 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -47,6 +47,7 @@ void initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt) { } static void trimEscape(SToken* pName) { + // todo need to deal with `ioo``ii` -> ioo`ii if (NULL != pName && pName->n > 1 && '`' == pName->z[0]) { pName->z += 1; pName->n -= 2; diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 435ab317e6..c9a0f4114e 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -377,35 +377,15 @@ static FORCE_INLINE int32_t checkAndTrimValue(SToken* pToken, uint32_t type, cha return buildSyntaxErrMsg(pMsgBuf, "invalid data or symbol", pToken->z); } - if (IS_NUMERIC_TYPE(type) && pToken->n == 0) { - return buildSyntaxErrMsg(pMsgBuf, "invalid numeric data", pToken->z); - } - // Remove quotation marks if (TK_NK_STRING == pToken->type) { if (pToken->n >= TSDB_MAX_BYTES_PER_ROW) { return buildSyntaxErrMsg(pMsgBuf, "too long string", pToken->z); } - // delete escape character: \\, \', \" - char delim = pToken->z[0]; - int32_t cnt = 0; - int32_t j = 0; - for (uint32_t k = 1; k < pToken->n - 1; ++k) { - if (pToken->z[k] == '\\' || (pToken->z[k] == delim && pToken->z[k + 1] == delim)) { - tmpTokenBuf[j] = pToken->z[k + 1]; - cnt++; - j++; - k++; - continue; - } - tmpTokenBuf[j] = pToken->z[k]; - j++; - } - - tmpTokenBuf[j] = 0; + int32_t len = trimString(pToken->z, pToken->n, tmpTokenBuf, TSDB_MAX_BYTES_PER_ROW); pToken->z = tmpTokenBuf; - pToken->n -= 2 + cnt; + pToken->n = len; } return TSDB_CODE_SUCCESS; @@ -582,6 +562,13 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int return func(pMsgBuf, pToken->z, pToken->n, param); } + case TSDB_DATA_TYPE_JSON: { + if(pToken->n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){ + return buildSyntaxErrMsg(pMsgBuf, "json string too long than 4095", pToken->z); + } + return func(pMsgBuf, pToken->z, pToken->n, param); + } + case TSDB_DATA_TYPE_TIMESTAMP: { int64_t tmpVal; if (parseTime(end, pToken, timePrec, &tmpVal, pMsgBuf) != TSDB_CODE_SUCCESS) { @@ -702,8 +689,10 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar); } - memset(&pColList->boundColumns[pColList->numOfBound], 0, - sizeof(col_id_t) * (pColList->numOfCols - pColList->numOfBound)); + if(pColList->numOfCols > pColList->numOfBound){ + memset(&pColList->boundColumns[pColList->numOfBound], 0, + sizeof(col_id_t) * (pColList->numOfCols - pColList->numOfBound)); + } return TSDB_CODE_SUCCESS; } @@ -720,22 +709,31 @@ static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void *value, int32_t len, voi int8_t type = pa->schema->type; int16_t colId = pa->schema->colId; + if(TSDB_DATA_TYPE_JSON == type){ + return parseJsontoTagData(value, pa->builder, pMsgBuf, colId); + } + + if (value == NULL) { // it is a null data + // tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NULL, value, false, pa->toffset, pa->colIdx); + return TSDB_CODE_SUCCESS; + } + if (TSDB_DATA_TYPE_BINARY == type) { STR_WITH_SIZE_TO_VARSTR(pa->buf, value, len); - tdAddColToKVRow(pa->builder, colId, type, pa->buf); + tdAddColToKVRow(pa->builder, colId, pa->buf, varDataTLen(pa->buf)); } else if (TSDB_DATA_TYPE_NCHAR == type) { // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' int32_t output = 0; if (!taosMbsToUcs4(value, len, (TdUcs4*)varDataVal(pa->buf), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) { char buf[512] = {0}; snprintf(buf, tListLen(buf), "%s", strerror(errno)); - return buildSyntaxErrMsg(pMsgBuf, buf, value);; + return buildSyntaxErrMsg(pMsgBuf, buf, value); } varDataSetLen(pa->buf, output); - tdAddColToKVRow(pa->builder, colId, type, pa->buf); + tdAddColToKVRow(pa->builder, colId, varDataTLen(pa->buf)); } else { - tdAddColToKVRow(pa->builder, colId, type, value); + tdAddColToKVRow(pa->builder, colId, value, TYPE_BYTES[type]); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index b3947b872e..bd461c7622 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1542,6 +1542,17 @@ static int32_t checkTableSmaOption(STranslateContext* pCxt, SCreateTableStmt* pS return TSDB_CODE_SUCCESS; } +static int32_t checkTableTags(STranslateContext* pCxt, SCreateTableStmt* pStmt) { + SNode* pNode; + FOREACH(pNode, pStmt->pTags) { + SColumnDefNode* pCol = (SColumnDefNode*)pNode; + if(pCol->dataType.type == TSDB_DATA_TYPE_JSON && LIST_LENGTH(pStmt->pTags) > 1){ + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_ONE_JSON_TAG); + } + } + return TSDB_CODE_SUCCESS; +} + static int32_t checkTableRollupOption(STranslateContext* pCxt, SNodeList* pFuncs) { if (NULL == pFuncs) { return TSDB_CODE_SUCCESS; @@ -1577,6 +1588,9 @@ static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt if (TSDB_CODE_SUCCESS == code) { code = checkRangeOption(pCxt, "delay", pStmt->pOptions->pDelay, TSDB_MIN_DB_DELAY, TSDB_MAX_DB_DELAY); } + if (TSDB_CODE_SUCCESS == code) { + code = checkTableTags(pCxt, pStmt); + } return code; } @@ -2893,11 +2907,19 @@ static int32_t addValToKVRow(STranslateContext* pCxt, SValueNode* pVal, const SS } SVariant var; valueNodeToVariant(pVal, &var); + if(pSchema->type == TSDB_DATA_TYPE_JSON){ + if(var.nLen > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){ + return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", var.pz); + } + return parseJsontoTagData(var.pz, pBuilder, &pCxt->msgBuf, pSchema->colId); + } + char tagVal[TSDB_MAX_TAGS_LEN] = {0}; int32_t code = taosVariantDump(&var, tagVal, pSchema->type, true); if (TSDB_CODE_SUCCESS == code) { - tdAddColToKVRow(pBuilder, pSchema->colId, pSchema->type, tagVal); + tdAddColToKVRow(pBuilder, pSchema->colId, tagVal, IS_VAR_DATA_TYPE(pSchema->type) ? varDataTLen(tagVal) : TYPE_BYTES[pSchema->type]); } + return code; } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 85d574d9e7..4ace62a2f1 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -14,6 +14,7 @@ */ #include "parUtil.h" +#include "cJSON.h" static char* getSyntaxErrFormat(int32_t errCode) { switch (errCode) { @@ -95,6 +96,8 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "Invalid option %s unit: %c, only m, h, d allowed"; case TSDB_CODE_PAR_INVALID_KEEP_UNIT: return "Invalid option keep unit: %c, %c, %c, only m, h, d allowed"; + case TSDB_CODE_PAR_ONLY_ONE_JSON_TAG: + return "Only one tag if there is a json tag"; case TSDB_CODE_OUT_OF_MEMORY: return "Out of memory"; default: @@ -195,24 +198,180 @@ STableComInfo getTableInfo(const STableMeta* pTableMeta) { } int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen) { - // delete escape character: \\, \', \" + if (len <=0 || dlen <= 0) return 0; + char delim = src[0]; - int32_t cnt = 0; int32_t j = 0; for (uint32_t k = 1; k < len - 1; ++k) { if (j >= dlen) { - break; + dst[j - 1] = '\0'; + return j; } - if (src[k] == '\\' || (src[k] == delim && src[k + 1] == delim)) { + if (src[k] == delim && src[k + 1] == delim) { // deal with "", '' dst[j] = src[k + 1]; - cnt++; j++; k++; continue; } + + if (src[k] == '\\') { // deal with escape character + if(src[k+1] == 'n'){ + dst[j] = '\n'; + }else if(src[k+1] == 'r'){ + dst[j] = '\r'; + }else if(src[k+1] == 't'){ + dst[j] = '\t'; + }else if(src[k+1] == '\\'){ + dst[j] = '\\'; + }else if(src[k+1] == '\''){ + dst[j] = '\''; + }else if(src[k+1] == '"'){ + dst[j] = '"'; + }else if(src[k+1] == '%' || src[k+1] == '_'){ + dst[j++] = src[k]; + dst[j] = src[k+1]; + }else{ + dst[j] = src[k+1]; + } + j++; + k++; + continue; + } + dst[j] = src[k]; j++; } dst[j] = '\0'; return j; } + +static bool isValidateTag(char *input) { + if (!input) return false; + for (size_t i = 0; i < strlen(input); ++i) { + if (isprint(input[i]) == 0) return false; + } + return true; +} + +int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* pMsgBuf, int16_t startColId){ + // set json NULL data + uint8_t jsonKeyNULL = TSDB_JSON_KEY_NULL; + uint8_t jsonNULL = TSDB_JSON_NULL; + int jsonIndex = startColId + 1; + tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonKeyNULL, CHAR_BYTES); // add json null type + if (!json || strtrim(json) == 0 || strncasecmp(json, "null", 4) == 0){ + tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNULL, CHAR_BYTES); // add json null value + return TSDB_CODE_SUCCESS; + } + + // set json real data + cJSON *root = cJSON_Parse(json); + if (root == NULL){ + return buildSyntaxErrMsg(pMsgBuf, "json parse error", json); + } + + int size = cJSON_GetArraySize(root); + if(!cJSON_IsObject(root)){ + return buildSyntaxErrMsg(pMsgBuf, "json error invalide value", json); + } + + int retCode = 0; + SHashObj* keyHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); + for(int i = 0; i < size; i++) { + cJSON* item = cJSON_GetArrayItem(root, i); + if (!item) { + qError("json inner error:%d", i); + return buildSyntaxErrMsg(pMsgBuf, "json inner error", json); + goto end; + } + + char *jsonKey = item->string; + if(!isValidateTag(jsonKey)){ + retCode = buildSyntaxErrMsg(pMsgBuf, "json key not validate", jsonKey); + goto end; + } +// if(strlen(jsonKey) > TSDB_MAX_JSON_KEY_LEN){ +// tscError("json key too long error"); +// retCode = tscSQLSyntaxErrMsg(errMsg, "json key too long, more than 256", NULL); +// goto end; +// } + size_t keyLen = strlen(jsonKey); + if(keyLen == 0 || taosHashGet(keyHash, jsonKey, keyLen) != NULL){ + continue; + } + + if(taosHashGetSize(keyHash) == 0){ + uint8_t jsonNotNULL = TSDB_JSON_NOT_NULL; + tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNotNULL, CHAR_BYTES); // add json type + } + // json key encode by binary + void *tagKey = taosMemoryCalloc(keyLen + VARSTR_HEADER_SIZE, 1); + if(!tagKey) { + retCode = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto end; + } + strncpy(varDataVal(tagKey), jsonKey, keyLen); + taosHashPut(keyHash, jsonKey, keyLen, &keyLen, CHAR_BYTES); // add key to hash to remove dumplicate, value is useless + + varDataSetLen(tagKey, keyLen); + tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKey, varDataTLen(tagKey)); // add json key + taosMemoryFree(tagKey); + + if(item->type == cJSON_String){ // add json value format: type|data + char *jsonValue = item->valuestring; + int32_t valLen = (int32_t)strlen(jsonValue); + char *tagVal = taosMemoryCalloc(valLen * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE + CHAR_BYTES, 1); + if(!tagVal) { + retCode = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto end; + } ; + tagVal[0] = TSDB_DATA_TYPE_NCHAR; + char* tagData = POINTER_SHIFT(tagVal, CHAR_BYTES); + if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)varDataVal(tagData), + (int32_t)(valLen * TSDB_NCHAR_SIZE), &valLen)) { + qError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, jsonValue, strerror(errno)); + retCode = buildSyntaxErrMsg(pMsgBuf, "charset convert json error", jsonValue); + taosMemoryFree(tagVal); + goto end; + } + + varDataSetLen(tagData, valLen); + tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagVal, CHAR_BYTES + varDataTLen(tagData)); + taosMemoryFree(tagVal); + }else if(item->type == cJSON_Number){ + if(!isfinite(item->valuedouble)){ + qError("json value is invalidate"); + retCode = buildSyntaxErrMsg(pMsgBuf, "json value number is illegal", json); + goto end; + } + char tagVal[LONG_BYTES + CHAR_BYTES] = {0}; + tagVal[0] = (item->valuedouble - (int64_t)(item->valuedouble) == 0) ? TSDB_DATA_TYPE_BIGINT + : TSDB_DATA_TYPE_DOUBLE; + char* tagData = POINTER_SHIFT(tagVal,CHAR_BYTES); + if(tagVal[0]== TSDB_DATA_TYPE_DOUBLE) *((double *)tagData) = item->valuedouble; + else if(tagVal[0] == TSDB_DATA_TYPE_BIGINT) *((int64_t *)tagData) = item->valueint; + tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagVal, LONG_BYTES + CHAR_BYTES); + }else if(item->type == cJSON_True || item->type == cJSON_False){ + char tagVal[CHAR_BYTES + CHAR_BYTES] = {0}; + tagVal[0] = TSDB_DATA_TYPE_BOOL; + tagVal[1] = (char)(item->valueint); + tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagVal, CHAR_BYTES + CHAR_BYTES); + }else if(item->type == cJSON_NULL){ + char tagVal[CHAR_BYTES] = {TSDB_DATA_TYPE_NULL}; + tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagVal, CHAR_BYTES); + } + else{ + retCode = buildSyntaxErrMsg(pMsgBuf, "invalidate json value", json); + goto end; + } + } + + if(taosHashGetSize(keyHash) == 0){ // set json NULL true + tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNULL, CHAR_BYTES); + } + +end: + taosHashCleanup(keyHash); + cJSON_Delete(root); + return retCode; +} \ No newline at end of file diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 191143de12..711ece3b01 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -57,7 +57,7 @@ OptrStr gOptrStr[] = { {OP_TYPE_IS_NOT_UNKNOWN, "not unknown"}, // json operator - {OP_TYPE_JSON_GET_VALUE, "json get"}, + {OP_TYPE_JSON_GET_VALUE, "->"}, {OP_TYPE_JSON_CONTAINS, "json contains"} }; diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index 93a843fee8..8df404f85f 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -276,3 +276,4 @@ int32_t tjsonToArray(const SJson* pJson, const char* pName, FToObject func, void } SJson* tjsonParse(const char* pStr) { return cJSON_Parse(pStr); } +bool tjsonIsObject(const SJson* pJson) { return cJSON_IsObject(pJson); } \ No newline at end of file diff --git a/source/util/src/tutil.c b/source/util/src/tutil.c index 8133e4d237..adb6a37ba7 100644 --- a/source/util/src/tutil.c +++ b/source/util/src/tutil.c @@ -47,65 +47,6 @@ int32_t strdequote(char *z) { return j + 1; // only one quote, do nothing } -int32_t strRmquote(char *z, int32_t len) { - // delete escape character: \\, \', \" - char delim = z[0]; - if (delim != '\'' && delim != '\"') { - return len; - } - - int32_t cnt = 0; - int32_t j = 0; - for (uint32_t k = 1; k < len - 1; ++k) { - if (z[k] == '\\' || (z[k] == delim && z[k + 1] == delim)) { - if (z[k] == '\\' && z[k + 1] == '_') { - // match '_' self - } else { - z[j] = z[k + 1]; - cnt++; - j++; - k++; - continue; - } - } - - z[j] = z[k]; - j++; - } - - z[j] = 0; - - return len - 2 - cnt; -} - -int32_t strndequote(char *dst, const char *z, int32_t len) { - assert(dst != NULL); - if (z == NULL || len == 0) { - return 0; - } - - int32_t quote = z[0]; - int32_t i = 1, j = 0; - - while (z[i] != 0) { - if (z[i] == quote) { - if (z[i + 1] == quote) { - dst[j++] = (char)quote; - i++; - } else { - dst[j++] = 0; - return (j - 1); - } - } else { - dst[j++] = z[i]; - } - - i++; - } - - return j + 1; // only one quote, do nothing -} - size_t strtrim(char *z) { int32_t i = 0; int32_t j = 0; diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index b89d517ad3..5a96a2a87d 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -226,63 +226,27 @@ int32_t shellRunCommand(TAOS *con, char *command) { } } - bool esc = false; - char quote = 0, *cmd = command, *p = command; + char quote = 0, *cmd = command; for (char c = *command++; c != 0; c = *command++) { - if (esc) { - switch (c) { - case 'n': - c = '\n'; - break; - case 'r': - c = '\r'; - break; - case 't': - c = '\t'; - break; - case 'G': - *p++ = '\\'; - break; - case '\'': - case '"': - if (quote) { - *p++ = '\\'; - } - break; - } - *p++ = c; - esc = false; + if (c == '\\' && (*command == '\'' || *command == '"' || *command == '`')) { + command ++; continue; } - if (c == '\\') { - if (quote != 0 && (*command == '_' || *command == '\\')) { - // DO nothing - } else { - esc = true; - continue; - } - } - if (quote == c) { quote = 0; - } else if (quote == 0 && (c == '\'' || c == '"')) { + } else if (quote == 0 && (c == '\'' || c == '"' || c == '`')) { quote = c; - } - - *p++ = c; - if (c == ';' && quote == 0) { - c = *p; - *p = 0; + } else if (c == ';' && quote == 0) { + c = *command; + *command = 0; if (shellRunSingleCommand(con, cmd) < 0) { return -1; } - *p = c; - p = cmd; + *command = c; + cmd = command; } } - - *p = 0; return shellRunSingleCommand(con, cmd); } @@ -573,19 +537,23 @@ static void shellPrintNChar(const char *str, int length, int width) { while (pos < length) { TdWchar wc; int bytes = taosMbToWchar(&wc, str + pos, MB_CUR_MAX); - if (bytes == 0) { + if (bytes <= 0) { break; } - pos += bytes; - if (pos > length) { + if (pos + bytes > length) { break; } - + int w = 0; #ifdef WINDOWS - int w = bytes; + w = bytes; #else - int w = taosWcharWidth(wc); + if(*(str + pos) == '\t' || *(str + pos) == '\n' || *(str + pos) == '\r'){ + w = bytes; + }else{ + w = taosWcharWidth(wc); + } #endif + pos += bytes; if (w <= 0) { continue; } @@ -679,6 +647,7 @@ static void printField(const char *val, TAOS_FIELD *field, int width, int32_t le break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_JSON: shellPrintNChar(val, length, width); break; case TSDB_DATA_TYPE_TIMESTAMP: From 807252356147847cb4b3018e2aee04a651e128ac Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 14 Apr 2022 19:01:11 +0800 Subject: [PATCH 02/47] refactor: add json tag function for sql parser --- source/libs/parser/src/parInsert.c | 2 +- source/libs/parser/src/parUtil.c | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 9bc81d8fba..7d2d928aa9 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -738,7 +738,7 @@ static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void *value, int32_t len, voi } varDataSetLen(pa->buf, output); - tdAddColToKVRow(pa->builder, colId, varDataTLen(pa->buf)); + tdAddColToKVRow(pa->builder, colId, pa->buf, varDataTLen(pa->buf)); } else { tdAddColToKVRow(pa->builder, colId, value, TYPE_BYTES[type]); } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 4ace62a2f1..a423643e42 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -242,6 +242,7 @@ int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen) { j++; } dst[j] = '\0'; + strtrim(dst); return j; } @@ -259,7 +260,7 @@ int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* p uint8_t jsonNULL = TSDB_JSON_NULL; int jsonIndex = startColId + 1; tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonKeyNULL, CHAR_BYTES); // add json null type - if (!json || strtrim(json) == 0 || strncasecmp(json, "null", 4) == 0){ + if (!json || strcasecmp(json, TSDB_DATA_NULL_STR_L) == 0){ tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNULL, CHAR_BYTES); // add json null value return TSDB_CODE_SUCCESS; } @@ -281,13 +282,13 @@ int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* p cJSON* item = cJSON_GetArrayItem(root, i); if (!item) { qError("json inner error:%d", i); - return buildSyntaxErrMsg(pMsgBuf, "json inner error", json); + retCode = buildSyntaxErrMsg(pMsgBuf, "json inner error", json); goto end; } char *jsonKey = item->string; if(!isValidateTag(jsonKey)){ - retCode = buildSyntaxErrMsg(pMsgBuf, "json key not validate", jsonKey); + retCode = buildSyntaxErrMsg(pMsgBuf, "json key not validate", jsonKey); goto end; } // if(strlen(jsonKey) > TSDB_MAX_JSON_KEY_LEN){ From 69f09fd0e36d2198d30dc3a177ab02d97a36c52c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 14 Apr 2022 20:14:52 +0800 Subject: [PATCH 03/47] stmt --- include/libs/nodes/querynodes.h | 21 ++-- include/libs/parser/parser.h | 6 ++ include/util/taoserror.h | 2 + source/client/inc/clientInt.h | 1 + source/client/inc/clientStmt.h | 38 ++++++- source/client/src/clientEnv.c | 1 + source/client/src/clientImpl.c | 5 +- source/client/src/clientMain.c | 135 +++++++++++++------------ source/client/src/clientStmt.c | 129 ++++++++++++++--------- source/libs/parser/inc/parInsertData.h | 2 +- source/libs/parser/src/parInsert.c | 68 ++++++++++++- source/libs/parser/src/parInsertData.c | 22 ++++ source/libs/parser/src/parser.c | 94 ++++++++++++++++- source/util/src/terror.c | 2 + 14 files changed, 394 insertions(+), 132 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 3e78da63de..5e6c5233f7 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -280,13 +280,22 @@ typedef struct SVgDataBlocks { char *pData; // SMsgDesc + SSubmitReq + SSubmitBlk + ... } SVgDataBlocks; +typedef struct SStmtDataCtx { + uint64_t tbUid; + SHashObj* pVgroupsHashObj; // global + SHashObj* pTableBlockHashObj; // global + SHashObj* pSubTableHashObj; // global + SArray* pTableDataBlocks; // global +} SStmtDataCtx; + typedef struct SVnodeModifOpStmt { - ENodeType nodeType; - ENodeType sqlNodeType; - SArray* pDataBlocks; // data block for each vgroup, SArray. - uint8_t payloadType; // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert - uint32_t insertType; // insert data from [file|sql statement| bound statement] - const char* sql; // current sql statement position + ENodeType nodeType; + ENodeType sqlNodeType; + SArray* pDataBlocks; // data block for each vgroup, SArray. + uint8_t payloadType; // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert + uint32_t insertType; // insert data from [file|sql statement| bound statement] + const char* sql; // current sql statement position + SStmtDataCtx stmtCtx; } SVnodeModifOpStmt; typedef struct SExplainOptions { diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index fef624288a..eea9da4e52 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -22,6 +22,11 @@ extern "C" { #include "querynodes.h" +typedef struct SStmtCallback { + TAOS_STMT* pStmt; + int32_t (*getTbNameFn)(TAOS_STMT*, char**); +} SStmtCallback; + typedef struct SParseContext { uint64_t requestId; int32_t acctId; @@ -34,6 +39,7 @@ typedef struct SParseContext { char *pMsg; // extended error message if exists to help identifying the problem in sql statement. int32_t msgLen; // max length of the msg struct SCatalog *pCatalog; + SStmtCallback *pStmtCb; } SParseContext; typedef struct SCmdMsgInfo { diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 60535e2f49..428dca96f8 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -132,6 +132,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_INVALID_JSON_TYPE TAOS_DEF_ERROR_CODE(0, 0x0222) #define TSDB_CODE_TSC_VALUE_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x0223) #define TSDB_CODE_TSC_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0X0224) +#define TSDB_CODE_TSC_STMT_STATUS_ERROR TAOS_DEF_ERROR_CODE(0, 0X0225) +#define TSDB_CODE_TSC_STMT_TBNAME_ERROR TAOS_DEF_ERROR_CODE(0, 0X0226) // mnode-common #define TSDB_CODE_MND_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0300) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 4e3299fd1b..b352b65bea 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -213,6 +213,7 @@ typedef struct SRequestObj { int32_t sqlLen; int64_t self; char* msgBuf; + int32_t msgBufLen; void* pInfo; // sql parse info, generated by parser module int32_t code; SArray* dbList; diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index c29361758d..3ff7c0006b 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -26,21 +26,51 @@ typedef enum { STMT_TYPE_QUERY, } STMT_TYPE; +typedef enum { + STMT_INIT = 1, + STMT_PREPARE, + STMT_SETTBNAME, + STMT_BIND, + STMT_BIND_COL, + STMT_ADD_BATCH, + STMT_EXECUTE +} STMT_STATUS; + typedef struct STscStmt { - STMT_TYPE type; - //int16_t last; - //STscObj* taos; - //SSqlObj* pSql; + STMT_TYPE type; + STMT_STATUS status; + STscObj* taos; + SRequestObj* pRequest; + SQuery* pQuery; + char* sql; + int32_t sqlLen; + char* tbName; + TAOS_BIND* bindTags; //SMultiTbStmt mtb; //SNormalStmt normal; //int numOfRows; } STscStmt; + #define STMT_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) #define STMT_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) #define STMT_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) +#define STMT_CHK_STATUS(_stmt, _status, _v) do { + switch (_status) { + case STMT_INIT: + if ((_stmt)->status != 0) return (_v); + break; + case STMT_PREPARE: + if ((_stmt)->status != STMT_INIT) STMT_ERR_RET(_v); + break; + case STMT_SETTBNAME: + break; + } +} while (0) + + TAOS_STMT *stmtInit(TAOS *taos); int stmtClose(TAOS_STMT *stmt); int stmtExec(TAOS_STMT *stmt); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 7f413bdabc..e0e2e9d2c8 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -186,6 +186,7 @@ void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t ty pRequest->pTscObj = pObj; pRequest->body.fp = fp; // not used it yet pRequest->msgBuf = taosMemoryCalloc(1, ERROR_MSG_BUF_DEFAULT_SIZE); + pRequest->msgBufLen = ERROR_MSG_BUF_DEFAULT_SIZE; tsem_init(&pRequest->body.rspSem, 0, 0); registerRequest(pRequest); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 0252dfe634..37597ba8e9 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -143,7 +143,7 @@ int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj* return TSDB_CODE_SUCCESS; } -int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery) { +int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtCallback* pStmtCb) { STscObj* pTscObj = pRequest->pTscObj; SParseContext cxt = { @@ -156,6 +156,7 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery) { .pMsg = pRequest->msgBuf, .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, .pTransporter = pTscObj->pAppInfo->pTransporter, + .pStmtCb = pStmtCb, }; cxt.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); @@ -289,7 +290,7 @@ SRequestObj* execQueryImpl(STscObj* pTscObj, const char* sql, int sqlLen) { int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest); if (TSDB_CODE_SUCCESS == code) { - code = parseSql(pRequest, false, &pQuery); + code = parseSql(pRequest, false, &pQuery, NULL); } if (TSDB_CODE_SUCCESS == code) { diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 84f9c7ac30..56132c3041 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -569,56 +569,6 @@ TAOS_STMT *taos_stmt_init(TAOS *taos) { return stmtInit(taos); } -int taos_stmt_close(TAOS_STMT *stmt) { - if (stmt == NULL) { - tscError("NULL parameter for %s", __FUNCTION__); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - - return stmtClose(stmt); -} - -int taos_stmt_execute(TAOS_STMT *stmt) { - if (stmt == NULL) { - tscError("NULL parameter for %s", __FUNCTION__); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - - return stmtExec(stmt); -} - -char *taos_stmt_errstr(TAOS_STMT *stmt) { - if (stmt == NULL) { - tscError("NULL parameter for %s", __FUNCTION__); - terrno = TSDB_CODE_INVALID_PARA; - return NULL; - } - - return stmtErrstr(stmt); -} - -int taos_stmt_affected_rows(TAOS_STMT *stmt) { - if (stmt == NULL) { - tscError("NULL parameter for %s", __FUNCTION__); - terrno = TSDB_CODE_INVALID_PARA; - return 0; - } - - return stmtAffectedRows(stmt); -} - -int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND *bind) { - if (stmt == NULL || bind == NULL) { - tscError("NULL parameter for %s", __FUNCTION__); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - - return stmtBind(stmt, bind); -} - int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length) { if (stmt == NULL || sql == NULL) { tscError("NULL parameter for %s", __FUNCTION__); @@ -649,6 +599,52 @@ int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name) { return stmtSetTbNameTags(stmt, name, NULL); } +int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND *bind) { + if (stmt == NULL || bind == NULL) { + tscError("NULL parameter for %s", __FUNCTION__); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + + return stmtBind(stmt, bind); +} + +int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { + if (stmt == NULL || bind == NULL) { + tscError("NULL parameter for %s", __FUNCTION__); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + + if (bind->num <= 0 || bind->num > INT16_MAX) { + tscError("invalid bind num %d", bind->num); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + + return stmtBindBatch(stmt, bind); +} + +int taos_stmt_add_batch(TAOS_STMT *stmt) { + if (stmt == NULL) { + tscError("NULL parameter for %s", __FUNCTION__); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + + return stmtAddBatch(stmt); +} + +int taos_stmt_execute(TAOS_STMT *stmt) { + if (stmt == NULL) { + tscError("NULL parameter for %s", __FUNCTION__); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + + return stmtExec(stmt); +} + int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert) { if (stmt == NULL || insert == NULL) { tscError("NULL parameter for %s", __FUNCTION__); @@ -669,16 +665,6 @@ int taos_stmt_num_params(TAOS_STMT *stmt, int *nums) { return stmtGetParamNum(stmt, nums); } -int taos_stmt_add_batch(TAOS_STMT *stmt) { - if (stmt == NULL) { - tscError("NULL parameter for %s", __FUNCTION__); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - - return stmtAddBatch(stmt); -} - TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt) { if (stmt == NULL) { tscError("NULL parameter for %s", __FUNCTION__); @@ -689,14 +675,37 @@ TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt) { return stmtUseResult(stmt); } -int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { - if (stmt == NULL || bind == NULL) { +char *taos_stmt_errstr(TAOS_STMT *stmt) { + if (stmt == NULL) { + tscError("NULL parameter for %s", __FUNCTION__); + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } + + return stmtErrstr(stmt); +} + +int taos_stmt_affected_rows(TAOS_STMT *stmt) { + if (stmt == NULL) { + tscError("NULL parameter for %s", __FUNCTION__); + terrno = TSDB_CODE_INVALID_PARA; + return 0; + } + + return stmtAffectedRows(stmt); +} + + + + +int taos_stmt_close(TAOS_STMT *stmt) { + if (stmt == NULL) { tscError("NULL parameter for %s", __FUNCTION__); terrno = TSDB_CODE_INVALID_PARA; return terrno; } - return stmtBindBatch(stmt, bind); + return stmtClose(stmt); } diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 8c4cff9251..a43ffdf31e 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -4,11 +4,23 @@ #include "clientStmt.h" #include "tdef.h" +int32_t stmtGetTbName(TAOS_STMT *stmt, char **tbName) { + STscStmt* pStmt = (STscStmt*)stmt; + + if (NULL == pStmt->tbName) { + tscError("no table name set"); + STMT_ERR_RET(TSDB_CODE_TSC_STMT_TBNAME_ERROR); + } + + *tbName = pStmt->tbName; + + return TSDB_CODE_SUCCESS; +} + TAOS_STMT *stmtInit(TAOS *taos) { STscObj* pObj = (STscObj*)taos; STscStmt* pStmt = NULL; -#if 0 pStmt = taosMemoryCalloc(1, sizeof(STscStmt)); if (pStmt == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -16,38 +28,64 @@ TAOS_STMT *stmtInit(TAOS *taos) { return NULL; } pStmt->taos = pObj; - - SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); - - if (pSql == NULL) { - free(pStmt); - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscError("failed to allocate memory for statement"); - return NULL; - } - - if (TSDB_CODE_SUCCESS != tscAllocPayload(&pSql->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) { - free(pSql); - free(pStmt); - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscError("failed to malloc payload buffer"); - return NULL; - } - - tsem_init(&pSql->rspSem, 0, 0); - pSql->signature = pSql; - pSql->pTscObj = pObj; - pSql->maxRetry = TSDB_MAX_REPLICA; - pStmt->pSql = pSql; - pStmt->last = STMT_INIT; - pStmt->numOfRows = 0; - registerSqlObj(pSql); -#endif + pStmt->status = STMT_INIT; return pStmt; } -int stmtClose(TAOS_STMT *stmt) { +int stmtPrepare(TAOS_STMT *stmt, const char *sql, unsigned long length) { + STscStmt* pStmt = (STscStmt*)stmt; + + STMT_CHK_STATUS(stmt, STMT_PREPARE, TSDB_CODE_TSC_STMT_STATUS_ERROR); + + pStmt->sql = strndup(sql, length); + pStmt->sqlLen = length; + + return TSDB_CODE_SUCCESS; +} + + +int stmtSetTbNameTags(TAOS_STMT *stmt, const char *tbName, TAOS_BIND *tags) { + STscStmt* pStmt = (STscStmt*)stmt; + + STMT_CHK_STATUS(stmt, STMT_SETTBNAME, TSDB_CODE_TSC_STMT_STATUS_ERROR); + + if (tbName) { + pStmt->tbName = strdup(tbName); + } + + if (tags) { + pStmt->bindTags = tags; + } + + return TSDB_CODE_SUCCESS; +} + +int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { + STscStmt* pStmt = (STscStmt*)stmt; + + STMT_CHK_STATUS(stmt, STMT_BIND, TSDB_CODE_TSC_STMT_STATUS_ERROR); + + if (NULL == pStmt->pRequest) { + SStmtCallback stmtCb = {.pStmt = stmt, .getTbNameFn = stmtGetTbName}; + + STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql, pStmt->sqlLen, &pStmt->pRequest)); + STMT_ERR_RET(parseSql(pStmt->pRequest, false, &pStmt->pQuery, &stmtCb)); + } + + qBindStmtData(pStmt->pQuery, bind, pStmt->pRequest->msgBuf, pStmt->pRequest->msgBufLen); + + return TSDB_CODE_SUCCESS; +} + + +int stmtAddBatch(TAOS_STMT *stmt) { + STscStmt* pStmt = (STscStmt*)stmt; + + STMT_CHK_STATUS(stmt, STMT_BIND, TSDB_CODE_TSC_STMT_STATUS_ERROR); + + qBuildStmtOutput(pStmt->pQuery); + return TSDB_CODE_SUCCESS; } @@ -55,26 +93,25 @@ int stmtExec(TAOS_STMT *stmt) { return TSDB_CODE_SUCCESS; } + +int stmtClose(TAOS_STMT *stmt) { + return TSDB_CODE_SUCCESS; +} + char *stmtErrstr(TAOS_STMT *stmt) { - return NULL; + STscStmt* pStmt = (STscStmt*)stmt; + + if (stmt == NULL) { + return (char*) tstrerror(terrno); + } + + return taos_errstr(pStmt->pRequest); } int stmtAffectedRows(TAOS_STMT *stmt) { return TSDB_CODE_SUCCESS; } -int stmtBind(TAOS_STMT *stmt, TAOS_BIND *bind) { - return TSDB_CODE_SUCCESS; -} - -int stmtPrepare(TAOS_STMT *stmt, const char *sql, unsigned long length) { - return TSDB_CODE_SUCCESS; -} - -int stmtSetTbNameTags(TAOS_STMT *stmt, const char *name, TAOS_BIND *tags) { - return TSDB_CODE_SUCCESS; -} - int stmtIsInsert(TAOS_STMT *stmt, int *insert) { return TSDB_CODE_SUCCESS; } @@ -83,17 +120,9 @@ int stmtGetParamNum(TAOS_STMT *stmt, int *nums) { return TSDB_CODE_SUCCESS; } -int stmtAddBatch(TAOS_STMT *stmt) { - return TSDB_CODE_SUCCESS; -} - TAOS_RES *stmtUseResult(TAOS_STMT *stmt) { return NULL; } -int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { - return TSDB_CODE_SUCCESS; -} - diff --git a/source/libs/parser/inc/parInsertData.h b/source/libs/parser/inc/parInsertData.h index ee17de50e0..b3440eff70 100644 --- a/source/libs/parser/inc/parInsertData.h +++ b/source/libs/parser/inc/parInsertData.h @@ -89,7 +89,7 @@ static FORCE_INLINE int32_t getExtendedRowSize(STableDataBlocks *pBlock) { (int32_t)TD_BITMAP_BYTES(pTableInfo->numOfColumns - 1); } -static FORCE_INLINE void getSTSRowAppendInfo(SSchema *pSchema, uint8_t rowType, SParsedDataColInfo *spd, col_id_t idx, +static FORCE_INLINE void getSTSRowAppendInfo(uint8_t rowType, SParsedDataColInfo *spd, col_id_t idx, int32_t *toffset, col_id_t *colIdx) { col_id_t schemaIdx = 0; if (IS_DATA_COL_ORDERED(spd)) { diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 9a0483cd33..9be821d321 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -63,6 +63,7 @@ typedef struct SInsertParseContext { SArray* pVgDataBlocks; // global int32_t totalNum; SVnodeModifOpStmt* pOutput; + SStmtCallback* pStmtCb; } SInsertParseContext; typedef int32_t (*_row_append_fn_t)(SMsgBuf* pMsgBuf, const void *value, int32_t len, void *param); @@ -453,8 +454,7 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int if (isNullStr(pToken)) { if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { - int64_t tmpVal = 0; - return func(pMsgBuf, &tmpVal, pSchema->bytes, param); + return buildSyntaxErrMsg(pMsgBuf, "primary timestamp should not be null", pToken->z); } return func(pMsgBuf, NULL, 0, param); @@ -872,8 +872,22 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, for (int i = 0; i < spd->numOfBound; ++i) { NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken); SSchema* pSchema = &schema[spd->boundColumns[i] - 1]; + + if (sToken.type == TK_NK_QUESTION) { + isParseBindParam = true; + if (NULL == pCxt->pStmtCb) { + return buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", sToken.z); + } + + continue; + } + + if (isParseBindParam) { + return buildInvalidOperationMsg(&pCxt->msg, "no mix usage for ? and values"); + } + param.schema = pSchema; - getSTSRowAppendInfo(schema, pBuilder->rowType, spd, i, ¶m.toffset, ¶m.colIdx); + getSTSRowAppendInfo(pBuilder->rowType, spd, i, ¶m.toffset, ¶m.colIdx); CHECK_CODE(parseValueToken(&pCxt->pSql, &sToken, pSchema, timePrec, tmpTokenBuf, MemRowAppend, ¶m, &pCxt->msg)); if (PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { @@ -1005,11 +1019,15 @@ static void destroyInsertParseContext(SInsertParseContext* pCxt) { // VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path // [...]; static int32_t parseInsertBody(SInsertParseContext* pCxt) { + int32_t tbNum = 0; + // for each table while (1) { destroyInsertParseContextForTable(pCxt); SToken sToken; + char *tbName = NULL; + // pSql -> tb_name ... NEXT_TOKEN(pCxt->pSql, sToken); @@ -1021,6 +1039,21 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { break; } + if (TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT) && tbNum > 0) { + return buildInvalidOperationMsg(&pCxt->msg, "single table allowed in one stmt");; + } + + if (TK_NK_QUESTION == sToken.type) { + if (pCxt->pStmtCb) { + CHECK_CODE((*pCxt->pStmtCb->getTbNameFn)(pCxt->pStmtCb->pStmt, &tbName)); + + sToken.z = tbName; + sToken.n = strlen(tbName); + } else { + return buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", sToken.z); + } + } + SToken tbnameToken = sToken; NEXT_TOKEN(pCxt->pSql, sToken); @@ -1046,6 +1079,8 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { // pSql -> (field1_value, ...) [(field1_value2, ...) ...] CHECK_CODE(parseValuesClause(pCxt, dataBuf)); pCxt->pOutput->insertType = TSDB_QUERY_TYPE_INSERT; + + tbNum++; continue; } @@ -1058,13 +1093,31 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { } // todo pCxt->pOutput->insertType = TSDB_QUERY_TYPE_FILE_INSERT; + + tbNum++; continue; } return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", sToken.z); } + + if (TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) { + pCxt->pOutput->stmtCtx.tbUid = pCxt->pTableMeta->uid; + pCxt->pOutput->stmtCtx.pVgroupsHashObj = pCxt->pVgroupsHashObj; + pCxt->pOutput->stmtCtx.pTableBlockHashObj = pCxt->pTableBlockHashObj; + pCxt->pOutput->stmtCtx.pSubTableHashObj = pCxt->pSubTableHashObj; + pCxt->pOutput->stmtCtx.pTableDataBlocks = pCxt->pTableDataBlocks; + + pCxt->pVgroupsHashObj = NULL; + pCxt->pTableBlockHashObj = NULL; + pCxt->pSubTableHashObj = NULL; + pCxt->pTableDataBlocks = NULL; + + return TSDB_CODE_SUCCESS; + } + // merge according to vgId - if (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT) && taosHashGetSize(pCxt->pTableBlockHashObj) > 0) { + if (taosHashGetSize(pCxt->pTableBlockHashObj) > 0) { CHECK_CODE(mergeTableDataBlocks(pCxt->pTableBlockHashObj, pCxt->pOutput->payloadType, &pCxt->pVgDataBlocks)); } return buildOutput(pCxt); @@ -1086,7 +1139,8 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { .pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false), .pSubTableHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, false), .totalNum = 0, - .pOutput = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT) + .pOutput = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT), + .pStmtCb = pContext->pStmtCb }; if (NULL == context.pVgroupsHashObj || NULL == context.pTableBlockHashObj || @@ -1094,6 +1148,10 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } + if (pContext->pStmtCb) { + TSDB_QUERY_SET_TYPE(context.pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT); + } + *pQuery = taosMemoryCalloc(1, sizeof(SQuery)); if (NULL == *pQuery) { return TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index 088b25d544..7d9adbd16c 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -506,6 +506,28 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p return TSDB_CODE_SUCCESS; } +int32_t allocateMemForSize(STableDataBlocks *pDataBlock, int32_t allSize) { + size_t remain = pDataBlock->nAllocSize - pDataBlock->size; + uint32_t nAllocSizeOld = pDataBlock->nAllocSize; + + // expand the allocated size + if (remain < allSize) { + pDataBlock->nAllocSize = (pDataBlock->size + allSize) * 1.5; + + char *tmp = taosMemoryRealloc(pDataBlock->pData, (size_t)pDataBlock->nAllocSize); + if (tmp != NULL) { + pDataBlock->pData = tmp; + memset(pDataBlock->pData + pDataBlock->size, 0, pDataBlock->nAllocSize - pDataBlock->size); + } else { + // do nothing, if allocate more memory failed + pDataBlock->nAllocSize = nAllocSizeOld; + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + } + + return TSDB_CODE_SUCCESS; +} + int32_t allocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows) { size_t remain = pDataBlock->nAllocSize - pDataBlock->size; const int factor = 5; diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index ebe76cc129..57f2d3801c 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -51,6 +51,98 @@ int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery) { return code; } +int32_t qBindStmtData(SQuery* pQuery, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen) { + SVnodeModifOpStmt *modifyNode = (SVnodeModifOpStmt *)pQuery->pRoot; + SStmtDataCtx *pCtx = &modifyNode->stmtCtx; + STableDataBlocks *pDataBlock = (STableDataBlocks**)taosHashGet(pCtx->pTableBlockHashObj, (const char*)&pCtx->tbUid, sizeof(pCtx->tbUid)); + if (NULL == pDataBlock) { + return TSDB_CODE_QRY_APP_ERROR; + } + + SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); + int32_t extendedRowSize = getExtendedRowSize(pDataBlock); + SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; + SRowBuilder* pBuilder = &pDataBlock->rowBuilder; + SMemParam param = {.rb = pBuilder}; + SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; + + CHECK_CODE(allocateMemForSize(pDataBlock, extendedRowSize * bind->num); + + for (int32_t r = 0; r < bind->num; ++r) { + STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size); // skip the SSubmitBlk header + tdSRowResetBuf(pBuilder, row); + + // 1. set the parsed value from sql string + for (int c = 0; c < spd->numOfBound; ++c) { + SSchema* pColSchema = &pSchema[spd->boundColumns[c] - 1]; + + param.schema = pColSchema; + getSTSRowAppendInfo(pBuilder->rowType, spd, c, ¶m.toffset, ¶m.colIdx); + + if (bind[c].is_null && bind[c].is_null[r]) { + CHECK_CODE(MemRowAppend(&pBuf, NULL, 0, ¶m)); + } else { + int32_t colLen = pColSchema->bytes; + if (IS_VAR_DATA_TYPE(pColSchema->type)) { + colLen = bind[c].length[r]; + } + + CHECK_CODE(MemRowAppend(&pBuf, (char *)bind[c].buffer + bind[c].buffer_length * r, colLen, ¶m)); + } + + if (PRIMARYKEY_TIMESTAMP_COL_ID == pColSchema->colId) { + TSKEY tsKey = TD_ROW_KEY(row); + checkTimestamp(pDataBlock, (const char *)&tsKey); + } + } + + // set the null value for the columns that do not assign values + if ((spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { + for (int32_t i = 0; i < spd->numOfCols; ++i) { + if (spd->cols[i].valStat == VAL_STAT_NONE) { // the primary TS key is not VAL_STAT_NONE + tdAppendColValToTpRow(pBuilder, TD_VTYPE_NONE, getNullValue(pSchema[i].type), true, pSchema[i].type, i, + spd->cols[i].toffset); + } + } + } + + pDataBlock->size += extendedRowSize; + } + + SSubmitBlk *pBlocks = (SSubmitBlk *)(pDataBlock->pData); + if (TSDB_CODE_SUCCESS != setBlockInfo(pBlocks, pDataBlock, bind->num)) { + return buildInvalidOperationMsg(&pBuf, "too many rows in sql, total number of rows should be less than 32767"); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t qBuildStmtOutput(SQuery* pQuery) { + SVnodeModifOpStmt *modifyNode = (SVnodeModifOpStmt *)pQuery->pRoot; + SStmtDataCtx *pCtx = &modifyNode->stmtCtx; + int32_t code = 0; + SInsertParseContext insertCtx = { + .pVgroupsHashObj = pCtx->pVgroupsHashObj; + .pTableBlockHashObj = pCtx->pTableBlockHashObj; + .pSubTableHashObj = pCtx->pSubTableHashObj; + .pTableDataBlocks = pCtx->pTableDataBlocks; + }; + + // merge according to vgId + if (taosHashGetSize(pCtx->pTableBlockHashObj) > 0) { + CHECK_CODE_GOTO(mergeTableDataBlocks(pCtx->pTableBlockHashObj, modifyNode->payloadType, &insertCtx.pVgDataBlocks), _return); + } + + code = buildOutput(&insertCtx); + +_return: + + destroyInsertParseContext(&insertCtx); + + return code; +} + + void qDestroyQuery(SQuery* pQueryNode) { if (NULL == pQueryNode) { return; @@ -68,4 +160,4 @@ void qDestroyQuery(SQuery* pQueryNode) { int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema) { return extractResultSchema(pRoot, numOfCols, pSchema); -} \ No newline at end of file +} diff --git a/source/util/src/terror.c b/source/util/src/terror.c index ded42365b6..6fe272de8a 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -138,6 +138,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_JSON, "Invalid JSON format") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_JSON_TYPE, "Invalid JSON data type") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_VALUE_OUT_OF_RANGE, "Value out of range") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_INPUT, "Invalid tsc input") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_STATUS_ERROR, "Stmt API usage error") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_TBNAME_ERROR, "Stmt table name not set") // mnode-common TAOS_DEFINE_ERROR(TSDB_CODE_MND_APP_ERROR, "Mnode internal error") From e3ea173046f5620ad3e8b4e89327c96474ac4f0e Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 15 Apr 2022 20:10:51 +0800 Subject: [PATCH 04/47] stmt --- include/libs/nodes/querynodes.h | 10 +- include/util/taoserror.h | 1 + source/client/inc/clientStmt.h | 14 ++ source/client/src/clientImpl.c | 24 +-- source/client/src/clientMain.c | 15 +- source/client/src/clientStmt.c | 241 +++++++++++++++++++++++++++-- source/libs/parser/inc/parInt.h | 1 + source/libs/parser/src/parInsert.c | 38 ++--- source/libs/parser/src/parser.c | 93 ++++++++++- source/util/src/terror.c | 1 + 10 files changed, 385 insertions(+), 53 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 5e6c5233f7..b07568b0a0 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -282,10 +282,12 @@ typedef struct SVgDataBlocks { typedef struct SStmtDataCtx { uint64_t tbUid; - SHashObj* pVgroupsHashObj; // global - SHashObj* pTableBlockHashObj; // global - SHashObj* pSubTableHashObj; // global - SArray* pTableDataBlocks; // global + uint64_t tbSuid; + int8_t tbType; + SParsedDataColInfo tags; + + SHashObj* pVgroupsHashObj; + SHashObj* pTableBlockHashObj; } SStmtDataCtx; typedef struct SVnodeModifOpStmt { diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 428dca96f8..ffd32a82d4 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -134,6 +134,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0X0224) #define TSDB_CODE_TSC_STMT_STATUS_ERROR TAOS_DEF_ERROR_CODE(0, 0X0225) #define TSDB_CODE_TSC_STMT_TBNAME_ERROR TAOS_DEF_ERROR_CODE(0, 0X0226) +#define TSDB_CODE_TSC_STMT_CLAUSE_ERROR TAOS_DEF_ERROR_CODE(0, 0X0227) // mnode-common #define TSDB_CODE_MND_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0300) diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index 3ff7c0006b..a8460a664c 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -30,22 +30,36 @@ typedef enum { STMT_INIT = 1, STMT_PREPARE, STMT_SETTBNAME, + STMT_FETCH_TAG_FIELDS, + STMT_FETCH_COL_FIELDS, STMT_BIND, STMT_BIND_COL, STMT_ADD_BATCH, STMT_EXECUTE } STMT_STATUS; + + typedef struct STscStmt { STMT_TYPE type; STMT_STATUS status; + bool autoCreate; + uint64_t runTimes; STscObj* taos; + SCatalog* pCatalog; + SHashObj* pTableDataBlocks; + SHashObj* pVgList; + + bool tbNeedParse; + bool tbReuse; SRequestObj* pRequest; SQuery* pQuery; char* sql; int32_t sqlLen; char* tbName; + SName sname; TAOS_BIND* bindTags; + //SMultiTbStmt mtb; //SNormalStmt normal; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 37597ba8e9..cb8d4ce42a 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -283,16 +283,9 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList return pRequest->code; } -SRequestObj* execQueryImpl(STscObj* pTscObj, const char* sql, int sqlLen) { - SRequestObj* pRequest = NULL; - SQuery* pQuery = NULL; +SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code) { SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); - int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest); - if (TSDB_CODE_SUCCESS == code) { - code = parseSql(pRequest, false, &pQuery, NULL); - } - if (TSDB_CODE_SUCCESS == code) { switch (pQuery->execMode) { case QUERY_EXEC_MODE_LOCAL: @@ -324,6 +317,19 @@ SRequestObj* execQueryImpl(STscObj* pTscObj, const char* sql, int sqlLen) { return pRequest; } + +SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen) { + SRequestObj* pRequest = NULL; + SQuery* pQuery = NULL; + + int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest); + if (TSDB_CODE_SUCCESS == code) { + code = parseSql(pRequest, false, &pQuery, NULL); + } + + return launchQueryImpl(pRequest, pQuery, code); +} + int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest) { SCatalog* pCatalog = NULL; int32_t code = 0; @@ -368,7 +374,7 @@ SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen) { int32_t code = 0; while (retryNum++ < REQUEST_MAX_TRY_TIMES) { - pRequest = execQueryImpl(pTscObj, sql, sqlLen); + pRequest = launchQuery(pTscObj, sql, sqlLen); if (TSDB_CODE_SUCCESS == pRequest->code || !NEED_CLIENT_HANDLE_ERROR(pRequest->code)) { break; } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 56132c3041..914f58c2e0 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -580,13 +580,22 @@ int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length) { } int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_BIND *tags) { - if (stmt == NULL || name == NULL || tags == NULL) { + if (stmt == NULL || name == NULL) { tscError("NULL parameter for %s", __FUNCTION__); terrno = TSDB_CODE_INVALID_PARA; return terrno; } - return stmtSetTbNameTags(stmt, name, tags); + int32_t code = stmtSetTbName(stmt, name); + if (code) { + return code; + } + + if (tags) { + return stmtSetTbTags(stmt, tags); + } + + return TSDB_CODE_SUCCESS; } int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name) { @@ -596,7 +605,7 @@ int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name) { return terrno; } - return stmtSetTbNameTags(stmt, name, NULL); + return stmtSetTbName(stmt, name); } int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND *bind) { diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index a43ffdf31e..2549e52099 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -7,6 +7,8 @@ int32_t stmtGetTbName(TAOS_STMT *stmt, char **tbName) { STscStmt* pStmt = (STscStmt*)stmt; + pStmt->type = STMT_TYPE_MULTI_INSERT; + if (NULL == pStmt->tbName) { tscError("no table name set"); STMT_ERR_RET(TSDB_CODE_TSC_STMT_TBNAME_ERROR); @@ -17,18 +19,151 @@ int32_t stmtGetTbName(TAOS_STMT *stmt, char **tbName) { return TSDB_CODE_SUCCESS; } +int32_t stmtParseSql(STscStmt* pStmt) { + SStmtCallback stmtCb = {.pStmt = pStmt, .getTbNameFn = stmtGetTbName}; + + STMT_ERR_RET(parseSql(pStmt->pRequest, false, &pStmt->pQuery, &stmtCb)); + + pStmt->tbNeedParse = false; + + switch (nodeType(pStmt->pQuery->pRoot)) { + case QUERY_NODE_VNODE_MODIF_STMT: + if (0 == pStmt->type) { + pStmt->type = STMT_TYPE_INSERT; + } + break; + case QUERY_NODE_SELECT_STMT: + pStmt->type = STMT_TYPE_QUERY; + break; + default: + tscError("not supported stmt type %d", nodeType(pStmt->pQuery->pRoot)); + STMT_ERR_RET(TSDB_CODE_TSC_STMT_CLAUSE_ERROR); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t stmtCloneBlock(STableDataBlocks** pDst, STableDataBlocks* pSrc) { + *pDst = (STableDataBlocks*)taosMemoryMalloc(sizeof(STableDataBlocks)); + if (NULL == *pDst) { + STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + memcpy(*pDst, pSrc, sizeof(STableDataBlocks)); + (*pDst)->cloned = true; + + (*pDst)->pData = NULL; + (*pDst)->ordered = true; + (*pDst)->prevTS = INT64_MIN; + (*pDst)->size = sizeof(SSubmitBlk); + (*pDst)->tsSource = -1; + + return TSDB_CODE_SUCCESS; +} + +int32_t stmtSaveTableDataBlock(STscStmt *pStmt) { + if (pStmt->type != STMT_TYPE_MULTI_INSERT) { + return TSDB_CODE_SUCCESS; + } + + SVnodeModifOpStmt *modifyNode = (SVnodeModifOpStmt *)pStmt->pQuery->pRoot; + SStmtDataCtx *pCtx = &modifyNode->stmtCtx; + + uint64_t uid; + if (TSDB_CHILD_TABLE == pCtx->tbType) { + uid = pCtx->tbSuid; + } else { + ASSERT(TSDB_NORMAL_TABLE == pCtx->tbType); + uid = pCtx->tbUid; + } + + if (taosHashGet(pStmt->pTableDataBlocks, &uid, sizeof(uid))) { + return TSDB_CODE_SUCCESS; + } + + ASSERT(1 == taosHashGetSize(pStmt->pTableDataBlocks)); + + STableDataBlocks** pSrc = taosHashIterate(pStmt->pTableDataBlocks, NULL); + STableDataBlocks* pDst = NULL; + + STMT_ERR_RET(stmtCloneBlock(&pDst, *pSrc)); + + taosHashPut(pStmt->pTableDataBlocks, &uid, sizeof(uid), &pDst, POINTER_BYTES); + + return TSDB_CODE_SUCCESS; +} + +int32_t stmtHandleTbInCache(STscStmt* pStmt) { + if (NULL == pStmt->pTableDataBlocks || taosHashGetSize(pStmt->pTableDataBlocks) <= 0) { + return TSDB_CODE_SUCCESS; + } + + if (NULL == pStmt->pCatalog) { + STMT_ERR_RET(catalogGetHandle(pStmt->taos->pAppInfo->clusterId, &pStmt->pCatalog)); + } + + STableMeta *pTableMeta = NULL; + SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp); + STMT_ERR_RET(catalogGetTableMeta(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->sname, &pTableMeta)); + + SVnodeModifOpStmt *modifyNode = (SVnodeModifOpStmt *)pStmt->pQuery->pRoot; + SStmtDataCtx *pCtx = &modifyNode->stmtCtx; + + if (pTableMeta->uid == pCtx->tbUid) { + pStmt->tbNeedParse = false; + pStmt->tbReuse = false; + return TSDB_CODE_SUCCESS; + } + + if (taosHashGet(pCtx->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid))) { + pStmt->tbNeedParse = false; + pStmt->tbReuse = true; + pCtx->tbUid = pTableMeta->uid; + pCtx->tbSuid = pTableMeta->suid; + pCtx->tbType = pTableMeta->tableType; + + return TSDB_CODE_SUCCESS; + } + + STableDataBlocks** pDataBlock = taosHashGet(pStmt->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid)) + if (pDataBlock && *pDataBlock) { + pStmt->tbNeedParse = false; + pStmt->tbReuse = true; + + pCtx->tbUid = pTableMeta->uid; + pCtx->tbSuid = pTableMeta->suid; + pCtx->tbType = pTableMeta->tableType; + + taosHashPut(pCtx->pTableBlockHashObj, &pCtx->tbUid, sizeof(pCtx->tbUid), pDataBlock, POINTER_BYTES); + + return TSDB_CODE_SUCCESS; + } + + return TSDB_CODE_SUCCESS; +} + + TAOS_STMT *stmtInit(TAOS *taos) { STscObj* pObj = (STscObj*)taos; STscStmt* pStmt = NULL; pStmt = taosMemoryCalloc(1, sizeof(STscStmt)); - if (pStmt == NULL) { + if (NULL == pStmt) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscError("failed to allocate memory for statement"); return NULL; } + + pStmt->pTableDataBlocks = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + pStmt->pVgList = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + if (NULL == pStmt->pTableDataBlocks || NULL == pStmt->pVgList) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + taosMemoryFree(pStmt); + return NULL; + } + pStmt->taos = pObj; pStmt->status = STMT_INIT; + pStmt->tbNeedParse = true; return pStmt; } @@ -45,34 +180,89 @@ int stmtPrepare(TAOS_STMT *stmt, const char *sql, unsigned long length) { } -int stmtSetTbNameTags(TAOS_STMT *stmt, const char *tbName, TAOS_BIND *tags) { +int stmtSetTbName(TAOS_STMT *stmt, const char *tbName, TAOS_BIND *tags) { STscStmt* pStmt = (STscStmt*)stmt; STMT_CHK_STATUS(stmt, STMT_SETTBNAME, TSDB_CODE_TSC_STMT_STATUS_ERROR); - if (tbName) { - pStmt->tbName = strdup(tbName); - } + taosMemoryFree(pStmt->tbName); - if (tags) { + if (NULL == pStmt->pRequest) { + STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql, pStmt->sqlLen, &pStmt->pRequest)); + } + + STMT_ERR_RET(qCreateSName(&pStmt->sname, tbName, pStmt->taos->acctId, pStmt->pRequest->pDb, pStmt->pRequest->msgBuf, pStmt->pRequest->msgBufLen)); + + pStmt->tbName = strdup(tbName); + + STMT_ERR_RET(stmtHandleTbInCache(pStmt)); + + return TSDB_CODE_SUCCESS; +} + +int stmtSetTbTags(TAOS_STMT *stmt, const char *tbName, TAOS_BIND *tags) { + STscStmt* pStmt = (STscStmt*)stmt; + + STMT_CHK_STATUS(stmt, STMT_SETTBNAME, TSDB_CODE_TSC_STMT_STATUS_ERROR); + + if (pStmt->tbNeedParse) { + taosMemoryFree(pStmt->bindTags); pStmt->bindTags = tags; + + STMT_ERR_RET(stmtParseSql(pStmt)); + } else { + //TODO BIND TAG DATA } return TSDB_CODE_SUCCESS; } + +int32_t stmtFetchTagFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fields) { + STscStmt* pStmt = (STscStmt*)stmt; + + STMT_CHK_STATUS(stmt, STMT_FETCH_TAG_FIELDS, TSDB_CODE_TSC_STMT_STATUS_ERROR); + + if (pStmt->tbNeedParse) { + STMT_ERR_RET(stmtParseSql(pStmt)); + } + + STMT_ERR_RET(qBuildStmtTagFields(pStmt->pQuery, fieldNum, fields)); + + return TSDB_CODE_SUCCESS; +} + +int32_t stmtFetchColFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD* fields) { + STscStmt* pStmt = (STscStmt*)stmt; + + STMT_CHK_STATUS(stmt, STMT_FETCH_COL_FIELDS, TSDB_CODE_TSC_STMT_STATUS_ERROR); + + if (pStmt->tbNeedParse) { + STMT_ERR_RET(stmtParseSql(pStmt)); + } + + STMT_ERR_RET(qBuildStmtColFields(pStmt->pQuery, fieldNum, fields)); + + return TSDB_CODE_SUCCESS; +} + int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { STscStmt* pStmt = (STscStmt*)stmt; STMT_CHK_STATUS(stmt, STMT_BIND, TSDB_CODE_TSC_STMT_STATUS_ERROR); - if (NULL == pStmt->pRequest) { - SStmtCallback stmtCb = {.pStmt = stmt, .getTbNameFn = stmtGetTbName}; - - STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql, pStmt->sqlLen, &pStmt->pRequest)); - STMT_ERR_RET(parseSql(pStmt->pRequest, false, &pStmt->pQuery, &stmtCb)); + if (pStmt->tbNeedParse && pStmt->runTimes && pStmt->type > 0 && STMT_TYPE_MULTI_INSERT != pStmt->type) { + pStmt->tbNeedParse = false; } + if (NULL == pStmt->pRequest) { + STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql, pStmt->sqlLen, &pStmt->pRequest)); + } + + if (pStmt->tbNeedParse) { + STMT_ERR_RET(stmtParseSql(pStmt)); + } + qBindStmtData(pStmt->pQuery, bind, pStmt->pRequest->msgBuf, pStmt->pRequest->msgBufLen); return TSDB_CODE_SUCCESS; @@ -82,15 +272,36 @@ int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { int stmtAddBatch(TAOS_STMT *stmt) { STscStmt* pStmt = (STscStmt*)stmt; - STMT_CHK_STATUS(stmt, STMT_BIND, TSDB_CODE_TSC_STMT_STATUS_ERROR); + STMT_CHK_STATUS(stmt, STMT_ADD_BATCH, TSDB_CODE_TSC_STMT_STATUS_ERROR); - qBuildStmtOutput(pStmt->pQuery); + STMT_ERR_RET(stmtSaveTableDataBlock(pStmt)); return TSDB_CODE_SUCCESS; } int stmtExec(TAOS_STMT *stmt) { - return TSDB_CODE_SUCCESS; + STscStmt* pStmt = (STscStmt*)stmt; + int32_t code = 0; + + STMT_CHK_STATUS(stmt, STMT_EXECUTE, TSDB_CODE_TSC_STMT_STATUS_ERROR); + + STMT_ERR_RET(qBuildStmtOutput(pStmt->pQuery)); + + launchQueryImpl(pStmt->pRequest, pStmt->pQuery, TSDB_CODE_SUCCESS); + + STMT_ERR_JRET(pStmt->pRequest->code); + +_return: + + //TODO RESET AND CLEAN PART TO DATABLOCK... + + taos_free_result(pStmt->pRequest); + pStmt->pRequest = NULL; + + pStmt->tbNeedParse = true; + ++pStmt->runTimes; + + STMT_RET(code); } diff --git a/source/libs/parser/inc/parInt.h b/source/libs/parser/inc/parInt.h index ea01caf9d7..8f88a05f77 100644 --- a/source/libs/parser/inc/parInt.h +++ b/source/libs/parser/inc/parInt.h @@ -27,6 +27,7 @@ int32_t parse(SParseContext* pParseCxt, SQuery** pQuery); int32_t translate(SParseContext* pParseCxt, SQuery* pQuery); int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema); int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery); +int32_t createSName(SName* pName, SToken* pTableName, int32_t acctId, char* dbName, SMsgBuf* pMsgBuf); #ifdef __cplusplus } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 9be821d321..d069208b23 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -59,7 +59,6 @@ typedef struct SInsertParseContext { SHashObj* pVgroupsHashObj; // global SHashObj* pTableBlockHashObj; // global SHashObj* pSubTableHashObj; // global - SArray* pTableDataBlocks; // global SArray* pVgDataBlocks; // global int32_t totalNum; SVnodeModifOpStmt* pOutput; @@ -164,7 +163,7 @@ static int32_t buildName(SInsertParseContext* pCxt, SToken* pStname, char* fullD return TSDB_CODE_SUCCESS; } -static int32_t createSName(SName* pName, SToken* pTableName, SParseContext* pParseCtx, SMsgBuf* pMsgBuf) { +int32_t createSName(SName* pName, SToken* pTableName, int32_t acctId, char* dbName, SMsgBuf* pMsgBuf) { const char* msg1 = "name too long"; const char* msg2 = "invalid database name"; const char* msg3 = "db is not specified"; @@ -180,7 +179,7 @@ static int32_t createSName(SName* pName, SToken* pTableName, SParseContext* pPar strncpy(name, pTableName->z, dbLen); dbLen = strdequote(name); - code = tNameSetDbName(pName, pParseCtx->acctId, name, dbLen); + code = tNameSetDbName(pName, acctId, name, dbLen); if (code != TSDB_CODE_SUCCESS) { return buildInvalidOperationMsg(pMsgBuf, msg1); } @@ -205,11 +204,11 @@ static int32_t createSName(SName* pName, SToken* pTableName, SParseContext* pPar strncpy(name, pTableName->z, pTableName->n); strdequote(name); - if (pParseCtx->db == NULL) { + if (dbName == NULL) { return buildInvalidOperationMsg(pMsgBuf, msg3); } - code = tNameSetDbName(pName, pParseCtx->acctId, pParseCtx->db, strlen(pParseCtx->db)); + code = tNameSetDbName(pName, acctId, dbName, strlen(dbName)); if (code != TSDB_CODE_SUCCESS) { code = buildInvalidOperationMsg(pMsgBuf, msg2); return code; @@ -227,7 +226,7 @@ static int32_t createSName(SName* pName, SToken* pTableName, SParseContext* pPar static int32_t getTableMetaImpl(SInsertParseContext* pCxt, SToken* pTname, bool isStb) { SParseContext* pBasicCtx = pCxt->pComCxt; SName name = {0}; - createSName(&name, pTname, pBasicCtx, &pCxt->msg); + createSName(&name, pTname, pBasicCtx->acctId, pBasicCtx->db, &pCxt->msg); if (isStb) { CHECK_CODE(catalogGetSTableMeta(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &pCxt->pTableMeta)); } else { @@ -812,7 +811,7 @@ static int32_t storeTableMeta(SHashObj* pHash, const char* pName, int32_t len, S // pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...) static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken) { SName name; - createSName(&name, pTbnameToken, pCxt->pComCxt, &pCxt->msg); + createSName(&name, pTbnameToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg); char tbFName[TSDB_TABLE_FNAME_LEN]; tNameExtractFullName(&name, tbFName); int32_t len = strlen(tbFName); @@ -1009,7 +1008,6 @@ static void destroyInsertParseContext(SInsertParseContext* pCxt) { taosHashCleanup(pCxt->pVgroupsHashObj); destroyBlockHashmap(pCxt->pTableBlockHashObj); - destroyBlockArrayList(pCxt->pTableDataBlocks); destroyBlockArrayList(pCxt->pVgDataBlocks); } @@ -1103,15 +1101,16 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { if (TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) { pCxt->pOutput->stmtCtx.tbUid = pCxt->pTableMeta->uid; + pCxt->pOutput->stmtCtx.tbSuid = pCxt->pTableMeta->suid; + pCxt->pOutput->stmtCtx.tbType = pCxt->pTableMeta->tableType; + pCxt->pOutput->stmtCtx.pVgroupsHashObj = pCxt->pVgroupsHashObj; pCxt->pOutput->stmtCtx.pTableBlockHashObj = pCxt->pTableBlockHashObj; - pCxt->pOutput->stmtCtx.pSubTableHashObj = pCxt->pSubTableHashObj; - pCxt->pOutput->stmtCtx.pTableDataBlocks = pCxt->pTableDataBlocks; + pCxt->pOutput->stmtCtx.tags = pCxt->tags; pCxt->pVgroupsHashObj = NULL; pCxt->pTableBlockHashObj = NULL; - pCxt->pSubTableHashObj = NULL; - pCxt->pTableDataBlocks = NULL; + memset(&pCxt->tags, 0, sizeof(pCxt->tags)); return TSDB_CODE_SUCCESS; } @@ -1152,14 +1151,17 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { TSDB_QUERY_SET_TYPE(context.pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT); } - *pQuery = taosMemoryCalloc(1, sizeof(SQuery)); if (NULL == *pQuery) { - return TSDB_CODE_OUT_OF_MEMORY; + *pQuery = taosMemoryCalloc(1, sizeof(SQuery)); + if (NULL == *pQuery) { + return TSDB_CODE_OUT_OF_MEMORY; + } + (*pQuery)->execMode = QUERY_EXEC_MODE_SCHEDULE; + (*pQuery)->haveResultSet = false; + (*pQuery)->msgType = TDMT_VND_SUBMIT; + (*pQuery)->pRoot = (SNode*)context.pOutput; } - (*pQuery)->execMode = QUERY_EXEC_MODE_SCHEDULE; - (*pQuery)->haveResultSet = false; - (*pQuery)->msgType = TDMT_VND_SUBMIT; - (*pQuery)->pRoot = (SNode*)context.pOutput; + context.pOutput->payloadType = PAYLOAD_TYPE_KV; int32_t code = skipInsertInto(&context); diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 57f2d3801c..edd797f35b 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -51,6 +51,33 @@ int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery) { return code; } +int32_t qCreateSName(SName* pName, char* pTableName, int32_t acctId, char* dbName, char *msgBuf, int32_t msgBufLen) { + SMsgBuf msg = {.buf = msgBuf, .len =msgBufLen}; + SToken sToken; + int32_t code = 0; + char *tbName = NULL; + + NEXT_TOKEN(pTableName, sToken); + + if (sToken.n == 0) { + return buildInvalidOperationMsg(&msg, "empty table name"); + } + + code = createSName(pName, &sToken, acctId, dbName, &msg); + if (code) { + return code; + } + + NEXT_TOKEN(pTableName, sToken); + + if (SToken.n > 0) { + return buildInvalidOperationMsg(&msg, "table name format is wrong"); + } + + return TSDB_CODE_SUCCESS; +} + + int32_t qBindStmtData(SQuery* pQuery, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen) { SVnodeModifOpStmt *modifyNode = (SVnodeModifOpStmt *)pQuery->pRoot; SStmtDataCtx *pCtx = &modifyNode->stmtCtx; @@ -122,10 +149,8 @@ int32_t qBuildStmtOutput(SQuery* pQuery) { SStmtDataCtx *pCtx = &modifyNode->stmtCtx; int32_t code = 0; SInsertParseContext insertCtx = { - .pVgroupsHashObj = pCtx->pVgroupsHashObj; - .pTableBlockHashObj = pCtx->pTableBlockHashObj; - .pSubTableHashObj = pCtx->pSubTableHashObj; - .pTableDataBlocks = pCtx->pTableDataBlocks; + .pVgroupsHashObj = pCtx->pVgroupsHashObj, + .pTableBlockHashObj = pCtx->pTableBlockHashObj, }; // merge according to vgId @@ -142,6 +167,66 @@ _return: return code; } +int32_t buildBoundFields(SParsedDataColInfo *boundInfo, SSchema *pSchema, int32_t *fieldNum, TAOS_FIELD** fields) { + *fields = taosMemoryCalloc(boundInfo->numOfBound, sizeof(TAOS_FIELD)); + if (NULL == *fields) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < boundInfo->numOfBound; ++i) { + SSchema* pTagSchema = &pSchema[boundInfo->boundColumns[i] - 1]; + strcpy((*fields)[i].name, pTagSchema->name); + (*fields)[i].type = pTagSchema->type; + (*fields)[i].bytes = pTagSchema->bytes; + } + + *fieldNum = boundInfo->numOfBound; + + return TSDB_CODE_SUCCESS; +} + + +int32_t qBuildStmtTagFields(SQuery* pQuery, int32_t *fieldNum, TAOS_FIELD** fields) { + SVnodeModifOpStmt *modifyNode = (SVnodeModifOpStmt *)pQuery->pRoot; + SStmtDataCtx *pCtx = &modifyNode->stmtCtx; + STableDataBlocks *pDataBlock = (STableDataBlocks**)taosHashGet(pCtx->pTableBlockHashObj, (const char*)&pCtx->tbUid, sizeof(pCtx->tbUid)); + if (NULL == pDataBlock) { + return TSDB_CODE_QRY_APP_ERROR; + } + + SSchema* pSchema = getTableTagSchema(pDataBlock->pTableMeta); + if (pCtx->tags.numOfBound <= 0) { + *fieldNum = 0; + *fields = NULL; + + return TSDB_CODE_SUCCESS; + } + + CHECK_CODE(buildBoundFields(&pCtx->tags, pSchema, fieldNum, fields)); + + return TSDB_CODE_SUCCESS; +} + +int32_t qBuildStmtColFields(SQuery* pQuery, int32_t *fieldNum, TAOS_FIELD** fields) { + SVnodeModifOpStmt *modifyNode = (SVnodeModifOpStmt *)pQuery->pRoot; + SStmtDataCtx *pCtx = &modifyNode->stmtCtx; + STableDataBlocks *pDataBlock = (STableDataBlocks**)taosHashGet(pCtx->pTableBlockHashObj, (const char*)&pCtx->tbUid, sizeof(pCtx->tbUid)); + if (NULL == pDataBlock) { + return TSDB_CODE_QRY_APP_ERROR; + } + + SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); + if (pCtx->tags.numOfBound <= 0) { + *fieldNum = 0; + *fields = NULL; + + return TSDB_CODE_SUCCESS; + } + + CHECK_CODE(buildBoundFields(&pDataBlock->boundColumnInfo, pSchema, fieldNum, fields)); + + return TSDB_CODE_SUCCESS; +} void qDestroyQuery(SQuery* pQueryNode) { if (NULL == pQueryNode) { diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 6fe272de8a..139b0d30f5 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -140,6 +140,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_VALUE_OUT_OF_RANGE, "Value out of range") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_INPUT, "Invalid tsc input") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_STATUS_ERROR, "Stmt API usage error") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_TBNAME_ERROR, "Stmt table name not set") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_CLAUSE_ERROR, "not supported stmt clause") // mnode-common TAOS_DEFINE_ERROR(TSDB_CODE_MND_APP_ERROR, "Mnode internal error") From 9547c0a09bcabed2b780f9554ebd4b435bc4ecdb Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sat, 16 Apr 2022 15:59:19 +0800 Subject: [PATCH 05/47] stmt --- include/libs/nodes/querynodes.h | 11 - include/libs/parser/parser.h | 5 +- include/util/taoserror.h | 2 +- source/client/inc/clientStmt.h | 56 ++-- source/client/src/clientImpl.c | 9 +- source/client/src/clientStmt.c | 337 +++++++++++++++++-------- source/libs/parser/src/parInsert.c | 25 +- source/libs/parser/src/parInsertData.c | 4 + source/libs/parser/src/parser.c | 52 ++-- source/util/src/terror.c | 2 +- 10 files changed, 321 insertions(+), 182 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index b07568b0a0..3407e597ff 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -280,16 +280,6 @@ typedef struct SVgDataBlocks { char *pData; // SMsgDesc + SSubmitReq + SSubmitBlk + ... } SVgDataBlocks; -typedef struct SStmtDataCtx { - uint64_t tbUid; - uint64_t tbSuid; - int8_t tbType; - SParsedDataColInfo tags; - - SHashObj* pVgroupsHashObj; - SHashObj* pTableBlockHashObj; -} SStmtDataCtx; - typedef struct SVnodeModifOpStmt { ENodeType nodeType; ENodeType sqlNodeType; @@ -297,7 +287,6 @@ typedef struct SVnodeModifOpStmt { uint8_t payloadType; // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert uint32_t insertType; // insert data from [file|sql statement| bound statement] const char* sql; // current sql statement position - SStmtDataCtx stmtCtx; } SVnodeModifOpStmt; typedef struct SExplainOptions { diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index eea9da4e52..f0fa977608 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -24,7 +24,10 @@ extern "C" { typedef struct SStmtCallback { TAOS_STMT* pStmt; - int32_t (*getTbNameFn)(TAOS_STMT*, char**); + int32_t (*getTbNameFn)(TAOS_STMT*, char**); + int32_t (*setBindInfoFn)(TAOS_STMT*, STableMeta*, void*); + int32_t (*setExecInfoFn)(TAOS_STMT*, SHashObj*, SHashObj*); + int32_t (*getExecInfoFn)(TAOS_STMT*, SHashObj**, SHashObj**); } SStmtCallback; typedef struct SParseContext { diff --git a/include/util/taoserror.h b/include/util/taoserror.h index ffd32a82d4..b97279cca2 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -132,7 +132,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_INVALID_JSON_TYPE TAOS_DEF_ERROR_CODE(0, 0x0222) #define TSDB_CODE_TSC_VALUE_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x0223) #define TSDB_CODE_TSC_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0X0224) -#define TSDB_CODE_TSC_STMT_STATUS_ERROR TAOS_DEF_ERROR_CODE(0, 0X0225) +#define TSDB_CODE_TSC_STMT_API_ERROR TAOS_DEF_ERROR_CODE(0, 0X0225) #define TSDB_CODE_TSC_STMT_TBNAME_ERROR TAOS_DEF_ERROR_CODE(0, 0X0226) #define TSDB_CODE_TSC_STMT_CLAUSE_ERROR TAOS_DEF_ERROR_CODE(0, 0X0227) diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index a8460a664c..7fb23840fe 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -38,27 +38,46 @@ typedef enum { STMT_EXECUTE } STMT_STATUS; +typedef struct SStmtTableCache { + STableDataBlocks* pDataBlock; + void* boundTags; +} SStmtTableCache; +typedef struct SStmtBindInfo { + bool needParse; + uint64_t tbUid; + uint64_t tbSuid; + int8_t tbType; + void* boundTags; + char* tbName; + SName sname; + TAOS_BIND* bindTags; +} SStmtBindInfo; -typedef struct STscStmt { +typedef struct SStmtExecInfo { + SRequestObj* pRequest; + SHashObj* pVgHash; + SHashObj* pBlockHash; +} SStmtExecInfo; + +typedef struct SStmtSQLInfo { STMT_TYPE type; STMT_STATUS status; bool autoCreate; uint64_t runTimes; - STscObj* taos; - SCatalog* pCatalog; - SHashObj* pTableDataBlocks; - SHashObj* pVgList; - - bool tbNeedParse; - bool tbReuse; - SRequestObj* pRequest; + SHashObj* pTableCache; //SHash SQuery* pQuery; - char* sql; + char* sqlStr; int32_t sqlLen; - char* tbName; - SName sname; - TAOS_BIND* bindTags; +} SStmtSQLInfo; + +typedef struct STscStmt { + STscObj* taos; + SCatalog* pCatalog; + + SStmtSQLInfo sql; + SStmtExecInfo exec; + SStmtBindInfo bind; //SMultiTbStmt mtb; //SNormalStmt normal; @@ -71,16 +90,19 @@ typedef struct STscStmt { #define STMT_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) #define STMT_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) -#define STMT_CHK_STATUS(_stmt, _status, _v) do { - switch (_status) { +#define STMT_SWITCH_STATUS(_stmt, _newstatus, _errcode) do { + switch (_newstatus) { case STMT_INIT: - if ((_stmt)->status != 0) return (_v); + if ((_stmt)->status != 0) return (_errcode); break; case STMT_PREPARE: - if ((_stmt)->status != STMT_INIT) STMT_ERR_RET(_v); + if ((_stmt)->status != STMT_INIT) STMT_ERR_RET(_errcode); break; case STMT_SETTBNAME: break; + default: + STMT_ERR_RET(_errcode); + break; } } while (0) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index cb8d4ce42a..93e79dfebb 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -283,7 +283,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList return pRequest->code; } -SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code) { +SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery) { SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); if (TSDB_CODE_SUCCESS == code) { @@ -309,7 +309,10 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code } taosArrayDestroy(pNodeList); - qDestroyQuery(pQuery); + if (!keepQuery) { + qDestroyQuery(pQuery); + } + if (NULL != pRequest && TSDB_CODE_SUCCESS != code) { pRequest->code = terrno; } @@ -327,7 +330,7 @@ SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen) { code = parseSql(pRequest, false, &pQuery, NULL); } - return launchQueryImpl(pRequest, pQuery, code); + return launchQueryImpl(pRequest, pQuery, code, false); } int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest) { diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 2549e52099..f2797adcc1 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -7,43 +7,92 @@ int32_t stmtGetTbName(TAOS_STMT *stmt, char **tbName) { STscStmt* pStmt = (STscStmt*)stmt; - pStmt->type = STMT_TYPE_MULTI_INSERT; + pStmt->sql.type = STMT_TYPE_MULTI_INSERT; - if (NULL == pStmt->tbName) { + if (NULL == pStmt->bind.tbName) { tscError("no table name set"); STMT_ERR_RET(TSDB_CODE_TSC_STMT_TBNAME_ERROR); } - *tbName = pStmt->tbName; + *tbName = pStmt->bind.tbName; + + return TSDB_CODE_SUCCESS; +} + +int32_t stmtSetBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags) { + STscStmt* pStmt = (STscStmt*)stmt; + + pStmt->bind.tbUid = pTableMeta->uid; + pStmt->bind.tbSuid = pTableMeta->suid; + pStmt->bind.tbType = pTableMeta->tableType; + pStmt->bind.boundTags = tags; + + return TSDB_CODE_SUCCESS; +} + +int32_t stmtSetExecInfo(TAOS_STMT* stmt, SHashObj* pVgHash, SHashObj* pBlockHash) { + STscStmt* pStmt = (STscStmt*)stmt; + + pStmt->exec.pVgHash = pVgHash; + pStmt->exec.pBlockHash = pBlockHash; + + return TSDB_CODE_SUCCESS; +} + +int32_t stmtGetExecInfo(TAOS_STMT* stmt, SHashObj** pVgHash, SHashObj** pBlockHash) { + STscStmt* pStmt = (STscStmt*)stmt; + + *pVgHash = pStmt->exec.pVgHash; + *pBlockHash = pStmt->exec.pBlockHash; return TSDB_CODE_SUCCESS; } int32_t stmtParseSql(STscStmt* pStmt) { - SStmtCallback stmtCb = {.pStmt = pStmt, .getTbNameFn = stmtGetTbName}; + SStmtCallback stmtCb = { + .pStmt = pStmt, + .getTbNameFn = stmtGetTbName, + .setBindInfoFn = stmtSetBindInfo, + .setExecInfoFn = stmtSetExecInfo, + .getExecInfoFn = stmtGetExecInfo, + }; - STMT_ERR_RET(parseSql(pStmt->pRequest, false, &pStmt->pQuery, &stmtCb)); + STMT_ERR_RET(parseSql(pStmt->exec.pRequest, false, &pStmt->sql.pQuery, &stmtCb)); - pStmt->tbNeedParse = false; + pStmt->bind.needParse = false; - switch (nodeType(pStmt->pQuery->pRoot)) { + switch (nodeType(pStmt->sql.pQuery->pRoot)) { case QUERY_NODE_VNODE_MODIF_STMT: - if (0 == pStmt->type) { - pStmt->type = STMT_TYPE_INSERT; + if (0 == pStmt->sql.type) { + pStmt->sql.type = STMT_TYPE_INSERT; } break; case QUERY_NODE_SELECT_STMT: - pStmt->type = STMT_TYPE_QUERY; + pStmt->sql.type = STMT_TYPE_QUERY; break; default: - tscError("not supported stmt type %d", nodeType(pStmt->pQuery->pRoot)); + tscError("not supported stmt type %d", nodeType(pStmt->sql.pQuery->pRoot)); STMT_ERR_RET(TSDB_CODE_TSC_STMT_CLAUSE_ERROR); } return TSDB_CODE_SUCCESS; } -int32_t stmtCloneBlock(STableDataBlocks** pDst, STableDataBlocks* pSrc) { +void stmtResetDataBlock(STableDataBlocks* pBlock) { + pBlock->pData = NULL; + pBlock->ordered = true; + pBlock->prevTS = INT64_MIN; + pBlock->size = sizeof(SSubmitBlk); + pBlock->tsSource = -1; + pBlock->numOfTables = 1; + pBlock->nAllocSize = 0; + pBlock->headerSize = pBlock->size; + + memset(&pBlock->rowBuilder, 0, sizeof(pBlock->rowBuilder)); +} + + +int32_t stmtCloneDataBlock(STableDataBlocks** pDst, STableDataBlocks* pSrc) { *pDst = (STableDataBlocks*)taosMemoryMalloc(sizeof(STableDataBlocks)); if (NULL == *pDst) { STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); @@ -52,49 +101,96 @@ int32_t stmtCloneBlock(STableDataBlocks** pDst, STableDataBlocks* pSrc) { memcpy(*pDst, pSrc, sizeof(STableDataBlocks)); (*pDst)->cloned = true; - (*pDst)->pData = NULL; - (*pDst)->ordered = true; - (*pDst)->prevTS = INT64_MIN; - (*pDst)->size = sizeof(SSubmitBlk); - (*pDst)->tsSource = -1; + stmtResetDataBlock(*pDst); return TSDB_CODE_SUCCESS; } -int32_t stmtSaveTableDataBlock(STscStmt *pStmt) { - if (pStmt->type != STMT_TYPE_MULTI_INSERT) { - return TSDB_CODE_SUCCESS; +void stmtFreeDataBlock(STableDataBlocks* pDataBlock) { + if (pDataBlock == NULL) { + return; } - SVnodeModifOpStmt *modifyNode = (SVnodeModifOpStmt *)pStmt->pQuery->pRoot; - SStmtDataCtx *pCtx = &modifyNode->stmtCtx; + taosMemoryFreeClear(pDataBlock->pData); + taosMemoryFreeClear(pDataBlock); +} + + +int32_t stmtCacheBlock(STscStmt *pStmt) { + if (pStmt->sql.type != STMT_TYPE_MULTI_INSERT) { + return TSDB_CODE_SUCCESS; + } uint64_t uid; - if (TSDB_CHILD_TABLE == pCtx->tbType) { - uid = pCtx->tbSuid; + if (TSDB_CHILD_TABLE == pStmt->bind.tbType) { + uid = pStmt->bind.tbSuid; } else { - ASSERT(TSDB_NORMAL_TABLE == pCtx->tbType); - uid = pCtx->tbUid; + ASSERT(TSDB_NORMAL_TABLE == pStmt->bind.tbType); + uid = pStmt->bind.tbUid; } - if (taosHashGet(pStmt->pTableDataBlocks, &uid, sizeof(uid))) { + if (taosHashGet(pStmt->sql.pTableCache, &uid, sizeof(uid))) { return TSDB_CODE_SUCCESS; } - ASSERT(1 == taosHashGetSize(pStmt->pTableDataBlocks)); - - STableDataBlocks** pSrc = taosHashIterate(pStmt->pTableDataBlocks, NULL); + STableDataBlocks** pSrc = taosHashGet(pStmt->exec.pBlockHash, &uid, sizeof(uid)); STableDataBlocks* pDst = NULL; - STMT_ERR_RET(stmtCloneBlock(&pDst, *pSrc)); + STMT_ERR_RET(stmtCloneDataBlock(&pDst, *pSrc)); - taosHashPut(pStmt->pTableDataBlocks, &uid, sizeof(uid), &pDst, POINTER_BYTES); + SStmtTableCache cache = { + .pDataBlock = pDst, + .boundTags = pStmt->bind.boundTags, + }; + + if (taosHashPut(pStmt->sql.pTableCache, &uid, sizeof(uid), &cache, sizeof(cache))) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pStmt->bind.boundTags = NULL; return TSDB_CODE_SUCCESS; } -int32_t stmtHandleTbInCache(STscStmt* pStmt) { - if (NULL == pStmt->pTableDataBlocks || taosHashGetSize(pStmt->pTableDataBlocks) <= 0) { +int32_t stmtCleanExecCtx(STscStmt* pStmt, bool keepTable) { + SVnodeModifOpStmt *modifyNode = (SVnodeModifOpStmt *)pStmt->sql.pQuery->pRoot; + + void *pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL); + while (pIter) { + STableDataBlocks* pBlocks = *(STableDataBlocks**)pIter; + + if (keepTable && (*(uint64_t*)taosHashGetKey(pIter, NULL) == pStmt->bind.tbUid)) { + taosMemoryFreeClear(pBlocks->pData); + stmtResetDataBlock(pBlocks); + + pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter); + continue; + } + + stmtFreeDataBlock(pBlocks); + + pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter); + } + + if (keepTable) { + return TSDB_CODE_SUCCESS; + } + + taosHashCleanup(pStmt->exec.pBlockHash); + pStmt->exec.pBlockHash = NULL; + + pStmt->bind.tbUid = 0; + pStmt->bind.tbSuid = 0; + pStmt->bind.tbType = 0; + + destroyBoundColumnInfo(pStmt->bind.boundTags); + taosMemoryFreeClear(pStmt->bind.boundTags); + + return TSDB_CODE_SUCCESS; +} + +int32_t stmtGetFromCache(STscStmt* pStmt) { + if (NULL == pStmt->sql.pTableCache || taosHashGetSize(pStmt->sql.pTableCache) <= 0) { return TSDB_CODE_SUCCESS; } @@ -104,37 +200,46 @@ int32_t stmtHandleTbInCache(STscStmt* pStmt) { STableMeta *pTableMeta = NULL; SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp); - STMT_ERR_RET(catalogGetTableMeta(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->sname, &pTableMeta)); + STMT_ERR_RET(catalogGetTableMeta(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->bind.sname, &pTableMeta)); - SVnodeModifOpStmt *modifyNode = (SVnodeModifOpStmt *)pStmt->pQuery->pRoot; - SStmtDataCtx *pCtx = &modifyNode->stmtCtx; - - if (pTableMeta->uid == pCtx->tbUid) { - pStmt->tbNeedParse = false; - pStmt->tbReuse = false; - return TSDB_CODE_SUCCESS; - } - - if (taosHashGet(pCtx->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid))) { - pStmt->tbNeedParse = false; - pStmt->tbReuse = true; - pCtx->tbUid = pTableMeta->uid; - pCtx->tbSuid = pTableMeta->suid; - pCtx->tbType = pTableMeta->tableType; + if (pTableMeta->uid == pStmt->bind.tbUid) { + pStmt->bind.needParse = false; return TSDB_CODE_SUCCESS; } - STableDataBlocks** pDataBlock = taosHashGet(pStmt->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid)) - if (pDataBlock && *pDataBlock) { - pStmt->tbNeedParse = false; - pStmt->tbReuse = true; + if (taosHashGet(pStmt->exec.pBlockHash, &pTableMeta->uid, sizeof(pTableMeta->uid))) { + SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &pTableMeta->uid, sizeof(pTableMeta->uid)); + if (NULL == pCache) { + tscError("table uid %" PRIx64 "found in exec blockHash, but not in sql blockHash", pTableMeta->uid); + STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR); + } + + pStmt->bind.needParse = false; + + pStmt->bind.tbUid = pTableMeta->uid; + pStmt->bind.tbSuid = pTableMeta->suid; + pStmt->bind.tbType = pTableMeta->tableType; + pStmt->bind.boundTags = pCache->boundTags; + + return TSDB_CODE_SUCCESS; + } - pCtx->tbUid = pTableMeta->uid; - pCtx->tbSuid = pTableMeta->suid; - pCtx->tbType = pTableMeta->tableType; + SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &pTableMeta->uid, sizeof(pTableMeta->uid)); + if (pCache) { + pStmt->bind.needParse = false; - taosHashPut(pCtx->pTableBlockHashObj, &pCtx->tbUid, sizeof(pCtx->tbUid), pDataBlock, POINTER_BYTES); + pStmt->bind.tbUid = pTableMeta->uid; + pStmt->bind.tbSuid = pTableMeta->suid; + pStmt->bind.tbType = pTableMeta->tableType; + pStmt->bind.boundTags = pCache->boundTags; + + STableDataBlocks* pNewBlock = NULL; + STMT_ERR_RET(stmtCloneDataBlock(&pNewBlock, pCache->pDataBlock)); + + if (taosHashPut(pStmt->exec.pBlockHash, &pStmt->bind.tbUid, sizeof(pStmt->bind.tbUid), &pNewBlock, POINTER_BYTES)) { + STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } return TSDB_CODE_SUCCESS; } @@ -153,17 +258,16 @@ TAOS_STMT *stmtInit(TAOS *taos) { return NULL; } - pStmt->pTableDataBlocks = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - pStmt->pVgList = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); - if (NULL == pStmt->pTableDataBlocks || NULL == pStmt->pVgList) { + pStmt->sql.pTableCache = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + if (NULL == pStmt->sql.pTableCache) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; taosMemoryFree(pStmt); return NULL; } pStmt->taos = pObj; - pStmt->status = STMT_INIT; - pStmt->tbNeedParse = true; + pStmt->sql.status = STMT_INIT; + pStmt->bind.needParse = true; return pStmt; } @@ -171,10 +275,10 @@ TAOS_STMT *stmtInit(TAOS *taos) { int stmtPrepare(TAOS_STMT *stmt, const char *sql, unsigned long length) { STscStmt* pStmt = (STscStmt*)stmt; - STMT_CHK_STATUS(stmt, STMT_PREPARE, TSDB_CODE_TSC_STMT_STATUS_ERROR); + STMT_SWITCH_STATUS(stmt, STMT_PREPARE, TSDB_CODE_TSC_STMT_API_ERROR); - pStmt->sql = strndup(sql, length); - pStmt->sqlLen = length; + pStmt->sql.sqlStr = strndup(sql, length); + pStmt->sql.sqlLen = length; return TSDB_CODE_SUCCESS; } @@ -183,19 +287,19 @@ int stmtPrepare(TAOS_STMT *stmt, const char *sql, unsigned long length) { int stmtSetTbName(TAOS_STMT *stmt, const char *tbName, TAOS_BIND *tags) { STscStmt* pStmt = (STscStmt*)stmt; - STMT_CHK_STATUS(stmt, STMT_SETTBNAME, TSDB_CODE_TSC_STMT_STATUS_ERROR); + STMT_SWITCH_STATUS(stmt, STMT_SETTBNAME, TSDB_CODE_TSC_STMT_API_ERROR); - taosMemoryFree(pStmt->tbName); + taosMemoryFree(pStmt->bind.tbName); - if (NULL == pStmt->pRequest) { - STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql, pStmt->sqlLen, &pStmt->pRequest)); + if (NULL == pStmt->exec.pRequest) { + STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest)); } - STMT_ERR_RET(qCreateSName(&pStmt->sname, tbName, pStmt->taos->acctId, pStmt->pRequest->pDb, pStmt->pRequest->msgBuf, pStmt->pRequest->msgBufLen)); + STMT_ERR_RET(qCreateSName(&pStmt->bind.sname, tbName, pStmt->taos->acctId, pStmt->exec.pRequest->pDb, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); - pStmt->tbName = strdup(tbName); + pStmt->bind.tbName = strdup(tbName); - STMT_ERR_RET(stmtHandleTbInCache(pStmt)); + STMT_ERR_RET(stmtGetFromCache(pStmt)); return TSDB_CODE_SUCCESS; } @@ -203,11 +307,11 @@ int stmtSetTbName(TAOS_STMT *stmt, const char *tbName, TAOS_BIND *tags) { int stmtSetTbTags(TAOS_STMT *stmt, const char *tbName, TAOS_BIND *tags) { STscStmt* pStmt = (STscStmt*)stmt; - STMT_CHK_STATUS(stmt, STMT_SETTBNAME, TSDB_CODE_TSC_STMT_STATUS_ERROR); + STMT_SWITCH_STATUS(stmt, STMT_SETTBNAME, TSDB_CODE_TSC_STMT_API_ERROR); - if (pStmt->tbNeedParse) { - taosMemoryFree(pStmt->bindTags); - pStmt->bindTags = tags; + if (pStmt->bind.needParse) { + taosMemoryFree(pStmt->bind.bindTags); + pStmt->bind.bindTags = tags; STMT_ERR_RET(stmtParseSql(pStmt)); } else { @@ -221,13 +325,24 @@ int stmtSetTbTags(TAOS_STMT *stmt, const char *tbName, TAOS_BIND *tags) { int32_t stmtFetchTagFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fields) { STscStmt* pStmt = (STscStmt*)stmt; - STMT_CHK_STATUS(stmt, STMT_FETCH_TAG_FIELDS, TSDB_CODE_TSC_STMT_STATUS_ERROR); + STMT_SWITCH_STATUS(stmt, STMT_FETCH_TAG_FIELDS, TSDB_CODE_TSC_STMT_API_ERROR); - if (pStmt->tbNeedParse) { + if (pStmt->bind.needParse) { STMT_ERR_RET(stmtParseSql(pStmt)); } - STMT_ERR_RET(qBuildStmtTagFields(pStmt->pQuery, fieldNum, fields)); + if (STMT_TYPE_QUERY == pStmt->sql.type) { + tscError("invalid operation to get query tag fileds"); + STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); + } + + STableDataBlocks *pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bind.tbUid, sizeof(pStmt->bind.tbUid)); + if (NULL == pDataBlock) { + tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bind.tbUid); + STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + STMT_ERR_RET(qBuildStmtTagFields(pDataBlock, pStmt->bind.boundTags, fieldNum, fields)); return TSDB_CODE_SUCCESS; } @@ -235,13 +350,24 @@ int32_t stmtFetchTagFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fiel int32_t stmtFetchColFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD* fields) { STscStmt* pStmt = (STscStmt*)stmt; - STMT_CHK_STATUS(stmt, STMT_FETCH_COL_FIELDS, TSDB_CODE_TSC_STMT_STATUS_ERROR); + STMT_SWITCH_STATUS(stmt, STMT_FETCH_COL_FIELDS, TSDB_CODE_TSC_STMT_API_ERROR); - if (pStmt->tbNeedParse) { + if (pStmt->bind.needParse) { STMT_ERR_RET(stmtParseSql(pStmt)); } - STMT_ERR_RET(qBuildStmtColFields(pStmt->pQuery, fieldNum, fields)); + if (STMT_TYPE_QUERY == pStmt->sql.type) { + tscError("invalid operation to get query column fileds"); + STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); + } + + STableDataBlocks *pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bind.tbUid, sizeof(pStmt->bind.tbUid)); + if (NULL == pDataBlock) { + tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bind.tbUid); + STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + STMT_ERR_RET(qBuildStmtColFields(pDataBlock, fieldNum, fields)); return TSDB_CODE_SUCCESS; } @@ -249,21 +375,27 @@ int32_t stmtFetchColFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD* field int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { STscStmt* pStmt = (STscStmt*)stmt; - STMT_CHK_STATUS(stmt, STMT_BIND, TSDB_CODE_TSC_STMT_STATUS_ERROR); + STMT_SWITCH_STATUS(stmt, STMT_BIND, TSDB_CODE_TSC_STMT_API_ERROR); - if (pStmt->tbNeedParse && pStmt->runTimes && pStmt->type > 0 && STMT_TYPE_MULTI_INSERT != pStmt->type) { - pStmt->tbNeedParse = false; + if (pStmt->bind.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { + pStmt->bind.needParse = false; } - if (NULL == pStmt->pRequest) { - STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql, pStmt->sqlLen, &pStmt->pRequest)); + if (NULL == pStmt->exec.pRequest) { + STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest)); } - if (pStmt->tbNeedParse) { + if (pStmt->bind.needParse) { STMT_ERR_RET(stmtParseSql(pStmt)); } + + STableDataBlocks *pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bind.tbUid, sizeof(pStmt->bind.tbUid)); + if (NULL == pDataBlock) { + tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bind.tbUid); + STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } - qBindStmtData(pStmt->pQuery, bind, pStmt->pRequest->msgBuf, pStmt->pRequest->msgBufLen); + qBindStmtData(pDataBlock, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen); return TSDB_CODE_SUCCESS; } @@ -272,9 +404,9 @@ int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { int stmtAddBatch(TAOS_STMT *stmt) { STscStmt* pStmt = (STscStmt*)stmt; - STMT_CHK_STATUS(stmt, STMT_ADD_BATCH, TSDB_CODE_TSC_STMT_STATUS_ERROR); + STMT_SWITCH_STATUS(stmt, STMT_ADD_BATCH, TSDB_CODE_TSC_STMT_API_ERROR); - STMT_ERR_RET(stmtSaveTableDataBlock(pStmt)); + STMT_ERR_RET(stmtCacheBlock(pStmt)); return TSDB_CODE_SUCCESS; } @@ -283,23 +415,24 @@ int stmtExec(TAOS_STMT *stmt) { STscStmt* pStmt = (STscStmt*)stmt; int32_t code = 0; - STMT_CHK_STATUS(stmt, STMT_EXECUTE, TSDB_CODE_TSC_STMT_STATUS_ERROR); + STMT_SWITCH_STATUS(stmt, STMT_EXECUTE, TSDB_CODE_TSC_STMT_API_ERROR); - STMT_ERR_RET(qBuildStmtOutput(pStmt->pQuery)); + STMT_ERR_RET(qBuildStmtOutput(pStmt->sql.pQuery, pStmt->exec.pVgHash, pStmt->exec.pBlockHash)); - launchQueryImpl(pStmt->pRequest, pStmt->pQuery, TSDB_CODE_SUCCESS); + launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, TSDB_CODE_SUCCESS, true); - STMT_ERR_JRET(pStmt->pRequest->code); + STMT_ERR_JRET(pStmt->exec.pRequest->code); _return: - //TODO RESET AND CLEAN PART TO DATABLOCK... + stmtCleanExecCtx(pStmt, (code ? false : true)); - taos_free_result(pStmt->pRequest); - pStmt->pRequest = NULL; + taos_free_result(pStmt->exec.pRequest); + pStmt->exec.pRequest = NULL; - pStmt->tbNeedParse = true; - ++pStmt->runTimes; + pStmt->bind.needParse = true; + + ++pStmt->sql.runTimes; STMT_RET(code); } @@ -316,7 +449,7 @@ char *stmtErrstr(TAOS_STMT *stmt) { return (char*) tstrerror(terrno); } - return taos_errstr(pStmt->pRequest); + return taos_errstr(pStmt->exec.pRequest); } int stmtAffectedRows(TAOS_STMT *stmt) { diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index d069208b23..6b7109fea3 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1100,17 +1100,17 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { } if (TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) { - pCxt->pOutput->stmtCtx.tbUid = pCxt->pTableMeta->uid; - pCxt->pOutput->stmtCtx.tbSuid = pCxt->pTableMeta->suid; - pCxt->pOutput->stmtCtx.tbType = pCxt->pTableMeta->tableType; - - pCxt->pOutput->stmtCtx.pVgroupsHashObj = pCxt->pVgroupsHashObj; - pCxt->pOutput->stmtCtx.pTableBlockHashObj = pCxt->pTableBlockHashObj; - pCxt->pOutput->stmtCtx.tags = pCxt->tags; + SParsedDataColInfo *tags = taosMemoryMalloc(sizeof(pCxt->tags)); + if (NULL == tags) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + memcpy(tags, &pCxt->tags, sizeof(pCxt->tags)); + (*pCxt->pStmtCb->setBindInfoFn)(pCxt->pTableMeta, tags); + memset(&pCxt->tags, 0, sizeof(pCxt->tags)); + (*pCxt->pStmtCb->setExecInfoFn)(pCxt->pStmtCb->pStmt, pCxt->pVgroupsHashObj, pCxt->pTableBlockHashObj); pCxt->pVgroupsHashObj = NULL; pCxt->pTableBlockHashObj = NULL; - memset(&pCxt->tags, 0, sizeof(pCxt->tags)); return TSDB_CODE_SUCCESS; } @@ -1134,14 +1134,19 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { .pSql = (char*) pContext->pSql, .msg = {.buf = pContext->pMsg, .len = pContext->msgLen}, .pTableMeta = NULL, - .pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false), - .pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false), .pSubTableHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, false), .totalNum = 0, .pOutput = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT), .pStmtCb = pContext->pStmtCb }; + if (pContext->pStmtCb && *pQuery) { + (*pContext->pStmtCb->getExecInfoFn)(pContext->pStmtCb->pStmt, &context.pVgroupsHashObj, &context.pTableBlockHashObj); + } else { + context.pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false), + context.pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false), + } + if (NULL == context.pVgroupsHashObj || NULL == context.pTableBlockHashObj || NULL == context.pSubTableHashObj || NULL == context.pOutput) { return TSDB_CODE_TSC_OUT_OF_MEMORY; diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index 7d9adbd16c..c601acf7bd 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -103,6 +103,10 @@ int32_t boundIdxCompar(const void *lhs, const void *rhs) { } void destroyBoundColumnInfo(SParsedDataColInfo* pColList) { + if (NULL == pColList) { + return; + } + taosMemoryFreeClear(pColList->boundColumns); taosMemoryFreeClear(pColList->cols); taosMemoryFreeClear(pColList->colIdxInfo); diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index edd797f35b..68e9961b6d 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -78,14 +78,7 @@ int32_t qCreateSName(SName* pName, char* pTableName, int32_t acctId, char* dbNam } -int32_t qBindStmtData(SQuery* pQuery, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen) { - SVnodeModifOpStmt *modifyNode = (SVnodeModifOpStmt *)pQuery->pRoot; - SStmtDataCtx *pCtx = &modifyNode->stmtCtx; - STableDataBlocks *pDataBlock = (STableDataBlocks**)taosHashGet(pCtx->pTableBlockHashObj, (const char*)&pCtx->tbUid, sizeof(pCtx->tbUid)); - if (NULL == pDataBlock) { - return TSDB_CODE_QRY_APP_ERROR; - } - +int32_t qBindStmtData(STableDataBlocks *pDataBlock, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen) { SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); int32_t extendedRowSize = getExtendedRowSize(pDataBlock); SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; @@ -144,27 +137,23 @@ int32_t qBindStmtData(SQuery* pQuery, TAOS_MULTI_BIND *bind, char *msgBuf, int32 return TSDB_CODE_SUCCESS; } -int32_t qBuildStmtOutput(SQuery* pQuery) { +int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash) { SVnodeModifOpStmt *modifyNode = (SVnodeModifOpStmt *)pQuery->pRoot; - SStmtDataCtx *pCtx = &modifyNode->stmtCtx; int32_t code = 0; SInsertParseContext insertCtx = { - .pVgroupsHashObj = pCtx->pVgroupsHashObj, - .pTableBlockHashObj = pCtx->pTableBlockHashObj, + .pVgroupsHashObj = pVgHash, + .pTableBlockHashObj = pBlockHash, + .pOutput = pQuery->pRoot }; // merge according to vgId - if (taosHashGetSize(pCtx->pTableBlockHashObj) > 0) { - CHECK_CODE_GOTO(mergeTableDataBlocks(pCtx->pTableBlockHashObj, modifyNode->payloadType, &insertCtx.pVgDataBlocks), _return); + if (taosHashGetSize(insertCtx.pTableBlockHashObj) > 0) { + CHECK_CODE_GOTO(mergeTableDataBlocks(insertCtx.pTableBlockHashObj, modifyNode->payloadType, &insertCtx.pVgDataBlocks), _return); } - - code = buildOutput(&insertCtx); -_return: + CHECK_CODE(buildOutput(&insertCtx)); - destroyInsertParseContext(&insertCtx); - - return code; + return TSDB_CODE_SUCCESS; } int32_t buildBoundFields(SParsedDataColInfo *boundInfo, SSchema *pSchema, int32_t *fieldNum, TAOS_FIELD** fields) { @@ -186,37 +175,28 @@ int32_t buildBoundFields(SParsedDataColInfo *boundInfo, SSchema *pSchema, int32_ } -int32_t qBuildStmtTagFields(SQuery* pQuery, int32_t *fieldNum, TAOS_FIELD** fields) { - SVnodeModifOpStmt *modifyNode = (SVnodeModifOpStmt *)pQuery->pRoot; - SStmtDataCtx *pCtx = &modifyNode->stmtCtx; - STableDataBlocks *pDataBlock = (STableDataBlocks**)taosHashGet(pCtx->pTableBlockHashObj, (const char*)&pCtx->tbUid, sizeof(pCtx->tbUid)); - if (NULL == pDataBlock) { +int32_t qBuildStmtTagFields(STableDataBlocks *pDataBlock, void *boundTags, int32_t *fieldNum, TAOS_FIELD** fields) { + SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; + if (NULL == tags) { return TSDB_CODE_QRY_APP_ERROR; } SSchema* pSchema = getTableTagSchema(pDataBlock->pTableMeta); - if (pCtx->tags.numOfBound <= 0) { + if (tags->numOfBound <= 0) { *fieldNum = 0; *fields = NULL; return TSDB_CODE_SUCCESS; } - CHECK_CODE(buildBoundFields(&pCtx->tags, pSchema, fieldNum, fields)); + CHECK_CODE(buildBoundFields(tags, pSchema, fieldNum, fields)); return TSDB_CODE_SUCCESS; } -int32_t qBuildStmtColFields(SQuery* pQuery, int32_t *fieldNum, TAOS_FIELD** fields) { - SVnodeModifOpStmt *modifyNode = (SVnodeModifOpStmt *)pQuery->pRoot; - SStmtDataCtx *pCtx = &modifyNode->stmtCtx; - STableDataBlocks *pDataBlock = (STableDataBlocks**)taosHashGet(pCtx->pTableBlockHashObj, (const char*)&pCtx->tbUid, sizeof(pCtx->tbUid)); - if (NULL == pDataBlock) { - return TSDB_CODE_QRY_APP_ERROR; - } - +int32_t qBuildStmtColFields(STableDataBlocks *pDataBlock, int32_t *fieldNum, TAOS_FIELD** fields) { SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); - if (pCtx->tags.numOfBound <= 0) { + if (pDataBlock->boundColumnInfo.numOfBound <= 0) { *fieldNum = 0; *fields = NULL; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 139b0d30f5..9332cb481e 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -138,7 +138,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_JSON, "Invalid JSON format") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_JSON_TYPE, "Invalid JSON data type") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_VALUE_OUT_OF_RANGE, "Value out of range") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_INPUT, "Invalid tsc input") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_STATUS_ERROR, "Stmt API usage error") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_API_ERROR, "Stmt API usage error") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_TBNAME_ERROR, "Stmt table name not set") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_CLAUSE_ERROR, "not supported stmt clause") From 1c499c94a71d6ee678fae31c5cbe1ca0d04125a4 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sat, 16 Apr 2022 17:47:42 +0800 Subject: [PATCH 06/47] stmt --- source/client/inc/clientStmt.h | 6 ---- source/client/src/clientStmt.c | 27 ++++++++++------ source/libs/parser/src/parInsert.c | 33 +++++++++++++++---- source/libs/parser/src/parser.c | 51 +++++++++++++++++++++++++++++- 4 files changed, 94 insertions(+), 23 deletions(-) diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index 7fb23840fe..cbfe380a03 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -51,7 +51,6 @@ typedef struct SStmtBindInfo { void* boundTags; char* tbName; SName sname; - TAOS_BIND* bindTags; } SStmtBindInfo; typedef struct SStmtExecInfo { @@ -78,11 +77,6 @@ typedef struct STscStmt { SStmtSQLInfo sql; SStmtExecInfo exec; SStmtBindInfo bind; - - //SMultiTbStmt mtb; - //SNormalStmt normal; - - //int numOfRows; } STscStmt; diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index f2797adcc1..54abeccc6a 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -85,7 +85,7 @@ void stmtResetDataBlock(STableDataBlocks* pBlock) { pBlock->size = sizeof(SSubmitBlk); pBlock->tsSource = -1; pBlock->numOfTables = 1; - pBlock->nAllocSize = 0; + pBlock->nAllocSize = TSDB_PAYLOAD_SIZE; pBlock->headerSize = pBlock->size; memset(&pBlock->rowBuilder, 0, sizeof(pBlock->rowBuilder)); @@ -237,6 +237,12 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { STableDataBlocks* pNewBlock = NULL; STMT_ERR_RET(stmtCloneDataBlock(&pNewBlock, pCache->pDataBlock)); + pNewBlock->pData = taosMemoryMalloc(pNewBlock->nAllocSize); + if (NULL == pNewBlock->pData) { + stmtFreeDataBlock(pNewBlock); + STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + if (taosHashPut(pStmt->exec.pBlockHash, &pStmt->bind.tbUid, sizeof(pStmt->bind.tbUid), &pNewBlock, POINTER_BYTES)) { STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -284,7 +290,7 @@ int stmtPrepare(TAOS_STMT *stmt, const char *sql, unsigned long length) { } -int stmtSetTbName(TAOS_STMT *stmt, const char *tbName, TAOS_BIND *tags) { +int stmtSetTbName(TAOS_STMT *stmt, const char *tbName) { STscStmt* pStmt = (STscStmt*)stmt; STMT_SWITCH_STATUS(stmt, STMT_SETTBNAME, TSDB_CODE_TSC_STMT_API_ERROR); @@ -304,20 +310,23 @@ int stmtSetTbName(TAOS_STMT *stmt, const char *tbName, TAOS_BIND *tags) { return TSDB_CODE_SUCCESS; } -int stmtSetTbTags(TAOS_STMT *stmt, const char *tbName, TAOS_BIND *tags) { +int stmtSetTbTags(TAOS_STMT *stmt, TAOS_BIND *tags) { STscStmt* pStmt = (STscStmt*)stmt; STMT_SWITCH_STATUS(stmt, STMT_SETTBNAME, TSDB_CODE_TSC_STMT_API_ERROR); if (pStmt->bind.needParse) { - taosMemoryFree(pStmt->bind.bindTags); - pStmt->bind.bindTags = tags; - STMT_ERR_RET(stmtParseSql(pStmt)); - } else { - //TODO BIND TAG DATA } + STableDataBlocks *pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bind.tbUid, sizeof(pStmt->bind.tbUid)); + if (NULL == pDataBlock) { + tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bind.tbUid); + STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + STMT_ERR_RET(qBindStmtTagsValue(pDataBlock, pStmt->bind.boundTags, pStmt->bind.tbSuid, &pStmt->bind.sname, tags, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); + return TSDB_CODE_SUCCESS; } @@ -395,7 +404,7 @@ int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - qBindStmtData(pDataBlock, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen); + qBindStmtColsValue(pDataBlock, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 6b7109fea3..e16e19b78a 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -753,14 +753,14 @@ static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void *value, int32_t len, voi return TSDB_CODE_SUCCESS; } -static int32_t buildCreateTbReq(SInsertParseContext* pCxt, const SName* pName, SKVRow row) { +static int32_t buildCreateTbReq(SVCreateTbReq *pTbReq, const SName* pName, SKVRow row, int64_t suid) { char dbFName[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pName, dbFName); - pCxt->createTblReq.type = TD_CHILD_TABLE; - pCxt->createTblReq.dbFName = strdup(dbFName); - pCxt->createTblReq.name = strdup(pName->tname); - pCxt->createTblReq.ctbCfg.suid = pCxt->pTableMeta->suid; - pCxt->createTblReq.ctbCfg.pTag = row; + pTbReq->type = TD_CHILD_TABLE; + pTbReq->dbFName = strdup(dbFName); + pTbReq->name = strdup(pName->tname); + pTbReq->ctbCfg.suid = suid; + pTbReq->ctbCfg.pTag = row; return TSDB_CODE_SUCCESS; } @@ -773,21 +773,40 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint SKvParam param = {.builder = &pCxt->tagsBuilder}; SToken sToken; + bool isParseBindParam = false; char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" for (int i = 0; i < pCxt->tags.numOfBound; ++i) { NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken); + + if (sToken.type == TK_NK_QUESTION) { + isParseBindParam = true; + if (NULL == pCxt->pStmtCb) { + return buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", sToken.z); + } + + continue; + } + + if (isParseBindParam) { + return buildInvalidOperationMsg(&pCxt->msg, "no mix usage for ? and tag values"); + } + SSchema* pTagSchema = &pSchema[pCxt->tags.boundColumns[i] - 1]; // colId starts with 1 param.schema = pTagSchema; CHECK_CODE(parseValueToken(&pCxt->pSql, &sToken, pTagSchema, precision, tmpTokenBuf, KvRowAppend, ¶m, &pCxt->msg)); } + if (isParseBindParam) { + return TSDB_CODE_SUCCESS; + } + SKVRow row = tdGetKVRowFromBuilder(&pCxt->tagsBuilder); if (NULL == row) { return buildInvalidOperationMsg(&pCxt->msg, "tag value expected"); } tdSortKVRowByColIdx(row); - return buildCreateTbReq(pCxt, pName, row); + return buildCreateTbReq(&pCxt->createTblReq, pName, row, pCxt->pTableMeta->suid); } static int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) { diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 68e9961b6d..5c57c873ee 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -77,8 +77,57 @@ int32_t qCreateSName(SName* pName, char* pTableName, int32_t acctId, char* dbNam return TSDB_CODE_SUCCESS; } +int32_t qBindStmtTagsValue(STableDataBlocks *pDataBlock, void *boundTags, int64_t suid, SName *pName, TAOS_BIND *bind, char *msgBuf, int32_t msgBufLen){ + SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; + SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; + if (NULL == tags) { + return TSDB_CODE_QRY_APP_ERROR; + } -int32_t qBindStmtData(STableDataBlocks *pDataBlock, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen) { + SKVRowBuilder tagBuilder; + if (tdInitKVRowBuilder(&tagBuilder) < 0) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SSchema* pSchema = getTableTagSchema(pDataBlock->pTableMeta); + SKvParam param = {.builder = &tagBuilder}; + + for (int c = 0; c < tags->numOfBound; ++c) { + if (bind[c].is_null && bind[c].is_null[0]) { + KvRowAppend(&pBuf, NULL, 0, ¶m); + continue; + } + + SSchema* pTagSchema = &pSchema[tags->boundColumns[c] - 1]; // colId starts with 1 + param.schema = pTagSchema; + + int32_t colLen = pTagSchema->bytes; + if (IS_VAR_DATA_TYPE(pTagSchema->type)) { + colLen = bind[c].length[0]; + } + + CHECK_CODE(KvRowAppend(&pBuf, (char *)bind[c].buffer, colLen, ¶m)); + } + + SKVRow row = tdGetKVRowFromBuilder(&tagBuilder); + if (NULL == row) { + tdDestroyKVRowBuilder(&tagBuilder); + return buildInvalidOperationMsg(&pBuf, "tag value expected"); + } + tdSortKVRowByColIdx(row); + + SVCreateTbReq tbReq = {0}; + CHECK_CODE(buildCreateTbReq(&tbReq, pName, row, suid)); + CHECK_CODE(buildCreateTbMsg(pDataBlock, &tbReq)); + + destroyCreateSubTbReq(&tbReq); + tdDestroyKVRowBuilder(&tagBuilder); + + return TSDB_CODE_SUCCESS; +} + + +int32_t qBindStmtColsValue(STableDataBlocks *pDataBlock, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen) { SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); int32_t extendedRowSize = getExtendedRowSize(pDataBlock); SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; From 93ca1168f3524c66c6a198aab754038109f15cab Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sat, 16 Apr 2022 19:41:18 +0800 Subject: [PATCH 07/47] stmt --- include/libs/parser/parser.h | 1 + source/client/inc/clientStmt.h | 36 +++-- source/client/src/clientMain.c | 14 +- source/client/src/clientStmt.c | 65 ++++++-- source/libs/parser/inc/parInsertData.h | 2 +- source/libs/parser/inc/parInt.h | 19 ++- source/libs/parser/src/parInsert.c | 68 ++++++-- source/libs/parser/src/parInsertData.c | 171 +++++++++++++++++++- source/libs/parser/src/parser.c | 206 ------------------------- 9 files changed, 324 insertions(+), 258 deletions(-) diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index f0fa977608..640b4181cd 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -21,6 +21,7 @@ extern "C" { #endif #include "querynodes.h" +#include "query.h" typedef struct SStmtCallback { TAOS_STMT* pStmt; diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index cbfe380a03..d84bbaeba1 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -23,7 +23,7 @@ extern "C" { typedef enum { STMT_TYPE_INSERT = 1, STMT_TYPE_MULTI_INSERT, - STMT_TYPE_QUERY, + STMT_TYPE_QUERY } STMT_TYPE; typedef enum { @@ -84,21 +84,24 @@ typedef struct STscStmt { #define STMT_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) #define STMT_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) -#define STMT_SWITCH_STATUS(_stmt, _newstatus, _errcode) do { - switch (_newstatus) { - case STMT_INIT: - if ((_stmt)->status != 0) return (_errcode); - break; - case STMT_PREPARE: - if ((_stmt)->status != STMT_INIT) STMT_ERR_RET(_errcode); - break; - case STMT_SETTBNAME: - break; - default: - STMT_ERR_RET(_errcode); - break; - } -} while (0) +#define STMT_SWITCH_STATUS(_stmt, _newstatus, _errcode) \ + do { \ + switch (_newstatus) { \ + case STMT_INIT: \ + if ((_stmt)->sql.status != 0) return (_errcode); \ + break; \ + case STMT_PREPARE: \ + if ((_stmt)->sql.status != STMT_INIT) STMT_ERR_RET(_errcode); \ + break; \ + case STMT_SETTBNAME: \ + break; \ + default: \ + STMT_ERR_RET(_errcode); \ + break; \ + } \ + \ + (_stmt)->sql.status = _newstatus; \ + } while (0) TAOS_STMT *stmtInit(TAOS *taos); @@ -106,7 +109,6 @@ int stmtClose(TAOS_STMT *stmt); int stmtExec(TAOS_STMT *stmt); char *stmtErrstr(TAOS_STMT *stmt); int stmtAffectedRows(TAOS_STMT *stmt); -int stmtBind(TAOS_STMT *stmt, TAOS_BIND *bind); int stmtPrepare(TAOS_STMT *stmt, const char *sql, unsigned long length); int stmtSetTbNameTags(TAOS_STMT *stmt, const char *name, TAOS_BIND *tags); int stmtIsInsert(TAOS_STMT *stmt, int *insert); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 914f58c2e0..91607b5fef 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -615,7 +615,15 @@ int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND *bind) { return terrno; } - return stmtBind(stmt, bind); + TAOS_MULTI_BIND mbind = {0}; + mbind.buffer_type = bind->buffer_type; + mbind.buffer = bind->buffer; + mbind.buffer_length = bind->buffer_length; + mbind.length = bind->length; + mbind.is_null = bind->is_null; + mbind.num = 1; + + return stmtBindBatch(stmt, &mbind); } int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { @@ -634,6 +642,10 @@ int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { return stmtBindBatch(stmt, bind); } +int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx) { + return stmtBindBatch(stmt, bind); /* TODO */ +} + int taos_stmt_add_batch(TAOS_STMT *stmt) { if (stmt == NULL) { tscError("NULL parameter for %s", __FUNCTION__); diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 54abeccc6a..2c0085a600 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -75,6 +75,8 @@ int32_t stmtParseSql(STscStmt* pStmt) { STMT_ERR_RET(TSDB_CODE_TSC_STMT_CLAUSE_ERROR); } + STMT_ERR_RET(stmtCacheBlock(pStmt)); + return TSDB_CODE_SUCCESS; } @@ -152,8 +154,20 @@ int32_t stmtCacheBlock(STscStmt *pStmt) { return TSDB_CODE_SUCCESS; } -int32_t stmtCleanExecCtx(STscStmt* pStmt, bool keepTable) { - SVnodeModifOpStmt *modifyNode = (SVnodeModifOpStmt *)pStmt->sql.pQuery->pRoot; +int32_t stmtCleanBindInfo(STscStmt* pStmt) { + pStmt->bind.tbUid = 0; + pStmt->bind.tbSuid = 0; + pStmt->bind.tbType = 0; + pStmt->bind.needParse = true; + + taosMemoryFreeClear(pStmt->bind.tbName); + destroyBoundColumnInfo(pStmt->bind.boundTags); + taosMemoryFreeClear(pStmt->bind.boundTags); +} + +int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable) { + taos_free_result(pStmt->exec.pRequest); + pStmt->exec.pRequest = NULL; void *pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL); while (pIter) { @@ -178,13 +192,33 @@ int32_t stmtCleanExecCtx(STscStmt* pStmt, bool keepTable) { taosHashCleanup(pStmt->exec.pBlockHash); pStmt->exec.pBlockHash = NULL; - - pStmt->bind.tbUid = 0; - pStmt->bind.tbSuid = 0; - pStmt->bind.tbType = 0; + + STMT_ERR_RET(stmtCleanBindInfo(pStmt)); + + return TSDB_CODE_SUCCESS; +} + +int32_t stmtCleanSQLInfo(STscStmt* pStmt) { + taosMemoryFree(pStmt->sql.sqlStr); + qDestroyQuery(pStmt->sql.pQuery); + + void *pIter = taosHashIterate(pStmt->sql.pTableCache, NULL); + while (pIter) { + SStmtTableCache* pCache = *(SStmtTableCache**)pIter; + + pCache->pDataBlock->cloned = false; + destroyDataBlock(pCache->pDataBlock); + destroyBoundColumnInfo(pCache->boundTags); - destroyBoundColumnInfo(pStmt->bind.boundTags); - taosMemoryFreeClear(pStmt->bind.boundTags); + pIter = taosHashIterate(pStmt->sql.pTableCache, pIter); + } + taosHashCleanup(pStmt->sql.pTableCache); + pStmt->sql.pTableCache = NULL; + + memset(&pStmt->sql, 0, sizeof(pStmt->sql)); + + STMT_ERR_RET(stmtCleanExecInfo(pStmt, false)); + STMT_ERR_RET(stmtCleanBindInfo(pStmt)); return TSDB_CODE_SUCCESS; } @@ -281,6 +315,10 @@ TAOS_STMT *stmtInit(TAOS *taos) { int stmtPrepare(TAOS_STMT *stmt, const char *sql, unsigned long length) { STscStmt* pStmt = (STscStmt*)stmt; + if (pStmt->sql.status >= STMT_PREPARE) { + STMT_ERR_RET(stmtCleanSQLInfo(pStmt)); + } + STMT_SWITCH_STATUS(stmt, STMT_PREPARE, TSDB_CODE_TSC_STMT_API_ERROR); pStmt->sql.sqlStr = strndup(sql, length); @@ -434,12 +472,7 @@ int stmtExec(TAOS_STMT *stmt) { _return: - stmtCleanExecCtx(pStmt, (code ? false : true)); - - taos_free_result(pStmt->exec.pRequest); - pStmt->exec.pRequest = NULL; - - pStmt->bind.needParse = true; + stmtCleanExecInfo(pStmt, (code ? false : true)); ++pStmt->sql.runTimes; @@ -466,6 +499,10 @@ int stmtAffectedRows(TAOS_STMT *stmt) { } int stmtIsInsert(TAOS_STMT *stmt, int *insert) { + STscStmt* pStmt = (STscStmt*)stmt; + + *insert = (STMT_TYPE_INSERT == pStmt->sql.type || STMT_TYPE_MULTI_INSERT == pStmt->sql.type); + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/inc/parInsertData.h b/source/libs/parser/inc/parInsertData.h index b3440eff70..f0fffff38b 100644 --- a/source/libs/parser/inc/parInsertData.h +++ b/source/libs/parser/inc/parInsertData.h @@ -131,7 +131,7 @@ static FORCE_INLINE int32_t setBlockInfo(SSubmitBlk *pBlocks, STableDataBlocks* int32_t schemaIdxCompar(const void *lhs, const void *rhs); int32_t boundIdxCompar(const void *lhs, const void *rhs); void setBoundColumnInfo(SParsedDataColInfo *pColList, SSchema *pSchema, col_id_t numOfCols); -void destroyBoundColumnInfo(SParsedDataColInfo* pColList); +void destroyBoundColumnInfo(void* pBoundInfo); void destroyBlockArrayList(SArray* pDataBlockList); void destroyBlockHashmap(SHashObj* pDataBlockHash); int initRowBuilder(SRowBuilder *pBuilder, int16_t schemaVer, SParsedDataColInfo *pColInfo); diff --git a/source/libs/parser/inc/parInt.h b/source/libs/parser/inc/parInt.h index 8f88a05f77..6b9b90c513 100644 --- a/source/libs/parser/inc/parInt.h +++ b/source/libs/parser/inc/parInt.h @@ -21,13 +21,30 @@ extern "C" { #endif #include "parser.h" +#include "parToken.h" +#include "parUtil.h" + +typedef struct SKvParam { + SKVRowBuilder *builder; + SSchema *schema; + char buf[TSDB_MAX_TAGS_LEN]; +} SKvParam; + +#define CHECK_CODE(expr) \ + do { \ + int32_t code = expr; \ + if (TSDB_CODE_SUCCESS != code) { \ + return code; \ + } \ + } while (0) int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery); int32_t parse(SParseContext* pParseCxt, SQuery** pQuery); int32_t translate(SParseContext* pParseCxt, SQuery* pQuery); int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema); int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery); -int32_t createSName(SName* pName, SToken* pTableName, int32_t acctId, char* dbName, SMsgBuf* pMsgBuf); +int32_t createSName(SName* pName, SToken* pTableName, int32_t acctId, const char* dbName, SMsgBuf* pMsgBuf); +int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void *value, int32_t len, void *param); #ifdef __cplusplus } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index e16e19b78a..4d89f70a59 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -40,14 +40,6 @@ sToken = tStrGetToken(pSql, &index, false); \ } while (0) -#define CHECK_CODE(expr) \ - do { \ - int32_t code = expr; \ - if (TSDB_CODE_SUCCESS != code) { \ - return code; \ - } \ - } while (0) - typedef struct SInsertParseContext { SParseContext* pComCxt; // input char *pSql; // input @@ -163,7 +155,8 @@ static int32_t buildName(SInsertParseContext* pCxt, SToken* pStname, char* fullD return TSDB_CODE_SUCCESS; } -int32_t createSName(SName* pName, SToken* pTableName, int32_t acctId, char* dbName, SMsgBuf* pMsgBuf) { + +int32_t createSName(SName* pName, SToken* pTableName, int32_t acctId, const char* dbName, SMsgBuf* pMsgBuf) { const char* msg1 = "name too long"; const char* msg2 = "invalid database name"; const char* msg3 = "db is not specified"; @@ -720,13 +713,7 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* return TSDB_CODE_SUCCESS; } -typedef struct SKvParam { - SKVRowBuilder *builder; - SSchema *schema; - char buf[TSDB_MAX_TAGS_LEN]; -} SKvParam; - -static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void *value, int32_t len, void *param) { +int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void *value, int32_t len, void *param) { SKvParam* pa = (SKvParam*) param; int8_t type = pa->schema->type; @@ -1195,3 +1182,52 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { destroyInsertParseContext(&context); return code; } + + +int32_t qCreateSName(SName* pName, char* pTableName, int32_t acctId, char* dbName, char *msgBuf, int32_t msgBufLen) { + SMsgBuf msg = {.buf = msgBuf, .len =msgBufLen}; + SToken sToken; + int32_t code = 0; + char *tbName = NULL; + + NEXT_TOKEN(pTableName, sToken); + + if (sToken.n == 0) { + return buildInvalidOperationMsg(&msg, "empty table name"); + } + + code = createSName(pName, &sToken, acctId, dbName, &msg); + if (code) { + return code; + } + + NEXT_TOKEN(pTableName, sToken); + + if (SToken.n > 0) { + return buildInvalidOperationMsg(&msg, "table name format is wrong"); + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash) { + SVnodeModifOpStmt *modifyNode = (SVnodeModifOpStmt *)pQuery->pRoot; + int32_t code = 0; + SInsertParseContext insertCtx = { + .pVgroupsHashObj = pVgHash, + .pTableBlockHashObj = pBlockHash, + .pOutput = pQuery->pRoot + }; + + // merge according to vgId + if (taosHashGetSize(insertCtx.pTableBlockHashObj) > 0) { + CHECK_CODE_GOTO(mergeTableDataBlocks(insertCtx.pTableBlockHashObj, modifyNode->payloadType, &insertCtx.pVgDataBlocks), _return); + } + + CHECK_CODE(buildOutput(&insertCtx)); + + return TSDB_CODE_SUCCESS; +} + + diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index c601acf7bd..9b31924c5e 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -18,6 +18,7 @@ #include "catalog.h" #include "parUtil.h" #include "querynodes.h" +#include "parInt.h" #define IS_RAW_PAYLOAD(t) \ (((int)(t)) == PAYLOAD_TYPE_RAW) // 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert @@ -102,10 +103,12 @@ int32_t boundIdxCompar(const void *lhs, const void *rhs) { } } -void destroyBoundColumnInfo(SParsedDataColInfo* pColList) { - if (NULL == pColList) { +void destroyBoundColumnInfo(void* pBoundInfo) { + if (NULL == pBoundInfo) { return; } + + SParsedDataColInfo* pColList = (SParsedDataColInfo*)pBoundInfo; taosMemoryFreeClear(pColList->boundColumns); taosMemoryFreeClear(pColList->cols); @@ -567,3 +570,167 @@ int initRowBuilder(SRowBuilder *pBuilder, int16_t schemaVer, SParsedDataColInfo pColInfo->boundNullLen); return TSDB_CODE_SUCCESS; } + +int32_t qBindStmtTagsValue(STableDataBlocks *pDataBlock, void *boundTags, int64_t suid, SName *pName, TAOS_BIND *bind, char *msgBuf, int32_t msgBufLen){ + SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; + SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; + if (NULL == tags) { + return TSDB_CODE_QRY_APP_ERROR; + } + + SKVRowBuilder tagBuilder; + if (tdInitKVRowBuilder(&tagBuilder) < 0) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SSchema* pSchema = getTableTagSchema(pDataBlock->pTableMeta); + SKvParam param = {.builder = &tagBuilder}; + + for (int c = 0; c < tags->numOfBound; ++c) { + if (bind[c].is_null && bind[c].is_null[0]) { + KvRowAppend(&pBuf, NULL, 0, ¶m); + continue; + } + + SSchema* pTagSchema = &pSchema[tags->boundColumns[c] - 1]; // colId starts with 1 + param.schema = pTagSchema; + + int32_t colLen = pTagSchema->bytes; + if (IS_VAR_DATA_TYPE(pTagSchema->type)) { + colLen = bind[c].length[0]; + } + + CHECK_CODE(KvRowAppend(&pBuf, (char *)bind[c].buffer, colLen, ¶m)); + } + + SKVRow row = tdGetKVRowFromBuilder(&tagBuilder); + if (NULL == row) { + tdDestroyKVRowBuilder(&tagBuilder); + return buildInvalidOperationMsg(&pBuf, "tag value expected"); + } + tdSortKVRowByColIdx(row); + + SVCreateTbReq tbReq = {0}; + CHECK_CODE(buildCreateTbReq(&tbReq, pName, row, suid)); + CHECK_CODE(buildCreateTbMsg(pDataBlock, &tbReq)); + + destroyCreateSubTbReq(&tbReq); + tdDestroyKVRowBuilder(&tagBuilder); + + return TSDB_CODE_SUCCESS; +} + + +int32_t qBindStmtColsValue(STableDataBlocks *pDataBlock, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen) { + SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); + int32_t extendedRowSize = getExtendedRowSize(pDataBlock); + SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; + SRowBuilder* pBuilder = &pDataBlock->rowBuilder; + SMemParam param = {.rb = pBuilder}; + SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; + + CHECK_CODE(allocateMemForSize(pDataBlock, extendedRowSize * bind->num); + + for (int32_t r = 0; r < bind->num; ++r) { + STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size); // skip the SSubmitBlk header + tdSRowResetBuf(pBuilder, row); + + // 1. set the parsed value from sql string + for (int c = 0; c < spd->numOfBound; ++c) { + SSchema* pColSchema = &pSchema[spd->boundColumns[c] - 1]; + + param.schema = pColSchema; + getSTSRowAppendInfo(pBuilder->rowType, spd, c, ¶m.toffset, ¶m.colIdx); + + if (bind[c].is_null && bind[c].is_null[r]) { + CHECK_CODE(MemRowAppend(&pBuf, NULL, 0, ¶m)); + } else { + int32_t colLen = pColSchema->bytes; + if (IS_VAR_DATA_TYPE(pColSchema->type)) { + colLen = bind[c].length[r]; + } + + CHECK_CODE(MemRowAppend(&pBuf, (char *)bind[c].buffer + bind[c].buffer_length * r, colLen, ¶m)); + } + + if (PRIMARYKEY_TIMESTAMP_COL_ID == pColSchema->colId) { + TSKEY tsKey = TD_ROW_KEY(row); + checkTimestamp(pDataBlock, (const char *)&tsKey); + } + } + + // set the null value for the columns that do not assign values + if ((spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { + for (int32_t i = 0; i < spd->numOfCols; ++i) { + if (spd->cols[i].valStat == VAL_STAT_NONE) { // the primary TS key is not VAL_STAT_NONE + tdAppendColValToTpRow(pBuilder, TD_VTYPE_NONE, getNullValue(pSchema[i].type), true, pSchema[i].type, i, + spd->cols[i].toffset); + } + } + } + + pDataBlock->size += extendedRowSize; + } + + SSubmitBlk *pBlocks = (SSubmitBlk *)(pDataBlock->pData); + if (TSDB_CODE_SUCCESS != setBlockInfo(pBlocks, pDataBlock, bind->num)) { + return buildInvalidOperationMsg(&pBuf, "too many rows in sql, total number of rows should be less than 32767"); + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t buildBoundFields(SParsedDataColInfo *boundInfo, SSchema *pSchema, int32_t *fieldNum, TAOS_FIELD** fields) { + *fields = taosMemoryCalloc(boundInfo->numOfBound, sizeof(TAOS_FIELD)); + if (NULL == *fields) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < boundInfo->numOfBound; ++i) { + SSchema* pTagSchema = &pSchema[boundInfo->boundColumns[i] - 1]; + strcpy((*fields)[i].name, pTagSchema->name); + (*fields)[i].type = pTagSchema->type; + (*fields)[i].bytes = pTagSchema->bytes; + } + + *fieldNum = boundInfo->numOfBound; + + return TSDB_CODE_SUCCESS; +} + + +int32_t qBuildStmtTagFields(STableDataBlocks *pDataBlock, void *boundTags, int32_t *fieldNum, TAOS_FIELD** fields) { + SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; + if (NULL == tags) { + return TSDB_CODE_QRY_APP_ERROR; + } + + SSchema* pSchema = getTableTagSchema(pDataBlock->pTableMeta); + if (tags->numOfBound <= 0) { + *fieldNum = 0; + *fields = NULL; + + return TSDB_CODE_SUCCESS; + } + + CHECK_CODE(buildBoundFields(tags, pSchema, fieldNum, fields)); + + return TSDB_CODE_SUCCESS; +} + +int32_t qBuildStmtColFields(STableDataBlocks *pDataBlock, int32_t *fieldNum, TAOS_FIELD** fields) { + SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); + if (pDataBlock->boundColumnInfo.numOfBound <= 0) { + *fieldNum = 0; + *fields = NULL; + + return TSDB_CODE_SUCCESS; + } + + CHECK_CODE(buildBoundFields(&pDataBlock->boundColumnInfo, pSchema, fieldNum, fields)); + + return TSDB_CODE_SUCCESS; +} + + diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 5c57c873ee..1674ef0ace 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -51,212 +51,6 @@ int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery) { return code; } -int32_t qCreateSName(SName* pName, char* pTableName, int32_t acctId, char* dbName, char *msgBuf, int32_t msgBufLen) { - SMsgBuf msg = {.buf = msgBuf, .len =msgBufLen}; - SToken sToken; - int32_t code = 0; - char *tbName = NULL; - - NEXT_TOKEN(pTableName, sToken); - - if (sToken.n == 0) { - return buildInvalidOperationMsg(&msg, "empty table name"); - } - - code = createSName(pName, &sToken, acctId, dbName, &msg); - if (code) { - return code; - } - - NEXT_TOKEN(pTableName, sToken); - - if (SToken.n > 0) { - return buildInvalidOperationMsg(&msg, "table name format is wrong"); - } - - return TSDB_CODE_SUCCESS; -} - -int32_t qBindStmtTagsValue(STableDataBlocks *pDataBlock, void *boundTags, int64_t suid, SName *pName, TAOS_BIND *bind, char *msgBuf, int32_t msgBufLen){ - SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; - SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; - if (NULL == tags) { - return TSDB_CODE_QRY_APP_ERROR; - } - - SKVRowBuilder tagBuilder; - if (tdInitKVRowBuilder(&tagBuilder) < 0) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SSchema* pSchema = getTableTagSchema(pDataBlock->pTableMeta); - SKvParam param = {.builder = &tagBuilder}; - - for (int c = 0; c < tags->numOfBound; ++c) { - if (bind[c].is_null && bind[c].is_null[0]) { - KvRowAppend(&pBuf, NULL, 0, ¶m); - continue; - } - - SSchema* pTagSchema = &pSchema[tags->boundColumns[c] - 1]; // colId starts with 1 - param.schema = pTagSchema; - - int32_t colLen = pTagSchema->bytes; - if (IS_VAR_DATA_TYPE(pTagSchema->type)) { - colLen = bind[c].length[0]; - } - - CHECK_CODE(KvRowAppend(&pBuf, (char *)bind[c].buffer, colLen, ¶m)); - } - - SKVRow row = tdGetKVRowFromBuilder(&tagBuilder); - if (NULL == row) { - tdDestroyKVRowBuilder(&tagBuilder); - return buildInvalidOperationMsg(&pBuf, "tag value expected"); - } - tdSortKVRowByColIdx(row); - - SVCreateTbReq tbReq = {0}; - CHECK_CODE(buildCreateTbReq(&tbReq, pName, row, suid)); - CHECK_CODE(buildCreateTbMsg(pDataBlock, &tbReq)); - - destroyCreateSubTbReq(&tbReq); - tdDestroyKVRowBuilder(&tagBuilder); - - return TSDB_CODE_SUCCESS; -} - - -int32_t qBindStmtColsValue(STableDataBlocks *pDataBlock, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen) { - SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); - int32_t extendedRowSize = getExtendedRowSize(pDataBlock); - SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; - SRowBuilder* pBuilder = &pDataBlock->rowBuilder; - SMemParam param = {.rb = pBuilder}; - SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; - - CHECK_CODE(allocateMemForSize(pDataBlock, extendedRowSize * bind->num); - - for (int32_t r = 0; r < bind->num; ++r) { - STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size); // skip the SSubmitBlk header - tdSRowResetBuf(pBuilder, row); - - // 1. set the parsed value from sql string - for (int c = 0; c < spd->numOfBound; ++c) { - SSchema* pColSchema = &pSchema[spd->boundColumns[c] - 1]; - - param.schema = pColSchema; - getSTSRowAppendInfo(pBuilder->rowType, spd, c, ¶m.toffset, ¶m.colIdx); - - if (bind[c].is_null && bind[c].is_null[r]) { - CHECK_CODE(MemRowAppend(&pBuf, NULL, 0, ¶m)); - } else { - int32_t colLen = pColSchema->bytes; - if (IS_VAR_DATA_TYPE(pColSchema->type)) { - colLen = bind[c].length[r]; - } - - CHECK_CODE(MemRowAppend(&pBuf, (char *)bind[c].buffer + bind[c].buffer_length * r, colLen, ¶m)); - } - - if (PRIMARYKEY_TIMESTAMP_COL_ID == pColSchema->colId) { - TSKEY tsKey = TD_ROW_KEY(row); - checkTimestamp(pDataBlock, (const char *)&tsKey); - } - } - - // set the null value for the columns that do not assign values - if ((spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { - for (int32_t i = 0; i < spd->numOfCols; ++i) { - if (spd->cols[i].valStat == VAL_STAT_NONE) { // the primary TS key is not VAL_STAT_NONE - tdAppendColValToTpRow(pBuilder, TD_VTYPE_NONE, getNullValue(pSchema[i].type), true, pSchema[i].type, i, - spd->cols[i].toffset); - } - } - } - - pDataBlock->size += extendedRowSize; - } - - SSubmitBlk *pBlocks = (SSubmitBlk *)(pDataBlock->pData); - if (TSDB_CODE_SUCCESS != setBlockInfo(pBlocks, pDataBlock, bind->num)) { - return buildInvalidOperationMsg(&pBuf, "too many rows in sql, total number of rows should be less than 32767"); - } - - return TSDB_CODE_SUCCESS; -} - -int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash) { - SVnodeModifOpStmt *modifyNode = (SVnodeModifOpStmt *)pQuery->pRoot; - int32_t code = 0; - SInsertParseContext insertCtx = { - .pVgroupsHashObj = pVgHash, - .pTableBlockHashObj = pBlockHash, - .pOutput = pQuery->pRoot - }; - - // merge according to vgId - if (taosHashGetSize(insertCtx.pTableBlockHashObj) > 0) { - CHECK_CODE_GOTO(mergeTableDataBlocks(insertCtx.pTableBlockHashObj, modifyNode->payloadType, &insertCtx.pVgDataBlocks), _return); - } - - CHECK_CODE(buildOutput(&insertCtx)); - - return TSDB_CODE_SUCCESS; -} - -int32_t buildBoundFields(SParsedDataColInfo *boundInfo, SSchema *pSchema, int32_t *fieldNum, TAOS_FIELD** fields) { - *fields = taosMemoryCalloc(boundInfo->numOfBound, sizeof(TAOS_FIELD)); - if (NULL == *fields) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < boundInfo->numOfBound; ++i) { - SSchema* pTagSchema = &pSchema[boundInfo->boundColumns[i] - 1]; - strcpy((*fields)[i].name, pTagSchema->name); - (*fields)[i].type = pTagSchema->type; - (*fields)[i].bytes = pTagSchema->bytes; - } - - *fieldNum = boundInfo->numOfBound; - - return TSDB_CODE_SUCCESS; -} - - -int32_t qBuildStmtTagFields(STableDataBlocks *pDataBlock, void *boundTags, int32_t *fieldNum, TAOS_FIELD** fields) { - SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; - if (NULL == tags) { - return TSDB_CODE_QRY_APP_ERROR; - } - - SSchema* pSchema = getTableTagSchema(pDataBlock->pTableMeta); - if (tags->numOfBound <= 0) { - *fieldNum = 0; - *fields = NULL; - - return TSDB_CODE_SUCCESS; - } - - CHECK_CODE(buildBoundFields(tags, pSchema, fieldNum, fields)); - - return TSDB_CODE_SUCCESS; -} - -int32_t qBuildStmtColFields(STableDataBlocks *pDataBlock, int32_t *fieldNum, TAOS_FIELD** fields) { - SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); - if (pDataBlock->boundColumnInfo.numOfBound <= 0) { - *fieldNum = 0; - *fields = NULL; - - return TSDB_CODE_SUCCESS; - } - - CHECK_CODE(buildBoundFields(&pDataBlock->boundColumnInfo, pSchema, fieldNum, fields)); - - return TSDB_CODE_SUCCESS; -} - void qDestroyQuery(SQuery* pQueryNode) { if (NULL == pQueryNode) { return; From b7bd654c773306a781045ed7f5b21705d71cdb97 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sat, 16 Apr 2022 21:24:07 +0800 Subject: [PATCH 08/47] refactor: add compare logic for json value --- include/common/ttypes.h | 2 +- include/util/tcompare.h | 2 + include/util/tdef.h | 1 - source/libs/nodes/src/nodesUtilFuncs.c | 2 +- source/libs/parser/src/parTranslater.c | 38 ++-- source/libs/parser/src/parUtil.c | 76 ++++--- source/libs/scalar/inc/sclvector.h | 4 + source/libs/scalar/src/filter.c | 9 +- source/libs/scalar/src/scalar.c | 77 ++++--- source/libs/scalar/src/sclvector.c | 283 +++++++++++++++++++++---- source/util/src/tcompare.c | 5 + 11 files changed, 358 insertions(+), 141 deletions(-) diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 19455db258..7c87235c39 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -49,7 +49,7 @@ typedef struct { #define varDataCopy(dst, v) memcpy((dst), (void *)(v), varDataTLen(v)) #define varDataLenByData(v) (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE)) #define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT)(_len)) -#define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR)) +#define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON)) #define varDataNetLen(v) (htons(((VarDataLenT *)(v))[0])) #define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v)) diff --git a/include/util/tcompare.h b/include/util/tcompare.h index cc9e8ae464..ed07ae1c9c 100644 --- a/include/util/tcompare.h +++ b/include/util/tcompare.h @@ -105,6 +105,8 @@ int32_t compareStrPatternNotMatch(const void *pLeft, const void *pRight); int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight); int32_t compareWStrPatternNotMatch(const void *pLeft, const void *pRight); +int32_t compareJsonContainsKey(const void *pLeft, const void *pRight); + __compar_fn_t getComparFunc(int32_t type, int32_t optr); __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order); int32_t doCompare(const char *a, const char *b, int32_t type, size_t size); diff --git a/include/util/tdef.h b/include/util/tdef.h index caf01f5a22..eacc3aa08e 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -274,7 +274,6 @@ typedef enum ELogicConditionType { #define TSDB_MAX_JSON_TAG_LEN 16384 #define TSDB_JSON_PLACEHOLDER 0x7F #define TSDB_JSON_null 0x00 -#define TSDB_JSON_KEY_NULL 0x00 #define TSDB_JSON_NOT_NULL 0x01 #define TSDB_JSON_NULL 0x00 diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 758f9d5d6f..1154894050 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -1006,6 +1006,7 @@ bool nodesIsComparisonOp(const SOperatorNode* pOp) { case OP_TYPE_NOT_LIKE: case OP_TYPE_MATCH: case OP_TYPE_NMATCH: + case OP_TYPE_JSON_CONTAINS: case OP_TYPE_IS_NULL: case OP_TYPE_IS_NOT_NULL: case OP_TYPE_IS_TRUE: @@ -1024,7 +1025,6 @@ bool nodesIsComparisonOp(const SOperatorNode* pOp) { bool nodesIsJsonOp(const SOperatorNode* pOp) { switch (pOp->opType) { case OP_TYPE_JSON_GET_VALUE: - case OP_TYPE_JSON_CONTAINS: return true; default: break; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index e0708e8cea..4a20d929b5 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -469,14 +469,18 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { pOp->node.resType.type = TSDB_DATA_TYPE_DOUBLE; pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; } else if (nodesIsComparisonOp(pOp)) { - if (TSDB_DATA_TYPE_JSON == ldt.type || TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type || + if (TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type || TSDB_DATA_TYPE_BLOB == rdt.type) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); } pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; - } else { - // todo json operator + } else if (nodesIsJsonOp(pOp)){ + if (TSDB_DATA_TYPE_JSON != ldt.type || TSDB_DATA_TYPE_BINARY != rdt.type) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); + } + pOp->node.resType.type = TSDB_DATA_TYPE_JSON; + pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_JSON].bytes; } return DEAL_RES_CONTINUE; } @@ -2814,25 +2818,25 @@ static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, c static int32_t addValToKVRow(STranslateContext* pCxt, SValueNode* pVal, const SSchema* pSchema, SKVRowBuilder* pBuilder) { + if(pSchema->type == TSDB_DATA_TYPE_JSON){ + if(pVal->literal && strlen(pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){ + return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pVal->literal); + } + + return parseJsontoTagData(pVal->literal, pBuilder, &pCxt->msgBuf, pSchema->colId); + } + if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) { return pCxt->errCode; } - SVariant var; - valueNodeToVariant(pVal, &var); - if(pSchema->type == TSDB_DATA_TYPE_JSON){ - if(var.nLen > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){ - return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", var.pz); - } - return parseJsontoTagData(var.pz, pBuilder, &pCxt->msgBuf, pSchema->colId); + + if(pVal->node.resType.type == TSDB_DATA_TYPE_NULL){ + // todo + }else{ + tdAddColToKVRow(pBuilder, pSchema->colId, &(pVal->datum.p), IS_VAR_DATA_TYPE(pSchema->type) ? varDataTLen(pVal->datum.p) : TYPE_BYTES[pSchema->type]); } - char tagVal[TSDB_MAX_TAGS_LEN] = {0}; - int32_t code = taosVariantDump(&var, tagVal, pSchema->type, true); - if (TSDB_CODE_SUCCESS == code) { - tdAddColToKVRow(pBuilder, pSchema->colId, tagVal, IS_VAR_DATA_TYPE(pSchema->type) ? varDataTLen(tagVal) : TYPE_BYTES[pSchema->type]); - } - - return code; + return TSDB_CODE_SUCCESS; } static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableClause* pStmt, STableMeta* pSuperTableMeta, diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 968b799816..2bb867e8d4 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -252,12 +252,10 @@ static bool isValidateTag(char *input) { int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* pMsgBuf, int16_t startColId){ // set json NULL data - uint8_t jsonKeyNULL = TSDB_JSON_KEY_NULL; uint8_t jsonNULL = TSDB_JSON_NULL; int jsonIndex = startColId + 1; - tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonKeyNULL, CHAR_BYTES); // add json null type if (!json || strcasecmp(json, TSDB_DATA_NULL_STR_L) == 0){ - tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNULL, CHAR_BYTES); // add json null value + tdAddColToKVRow(kvRowBuilder, jsonIndex, &jsonNULL, CHAR_BYTES); return TSDB_CODE_SUCCESS; } @@ -273,6 +271,7 @@ int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* p } int retCode = 0; + char *tagKV = NULL; SHashObj* keyHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); for(int i = 0; i < size; i++) { cJSON* item = cJSON_GetArrayItem(root, i); @@ -296,66 +295,64 @@ int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* p if(keyLen == 0 || taosHashGet(keyHash, jsonKey, keyLen) != NULL){ continue; } - + // key: keyLen + VARSTR_HEADER_SIZE, value type: CHAR_BYTES, value reserved: LONG_BYTES + tagKV = taosMemoryCalloc(keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES + LONG_BYTES, 1); + if(!tagKV) { + retCode = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto end; + } + strncpy(varDataVal(tagKV), jsonKey, keyLen); + varDataSetLen(tagKV, keyLen); if(taosHashGetSize(keyHash) == 0){ uint8_t jsonNotNULL = TSDB_JSON_NOT_NULL; tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNotNULL, CHAR_BYTES); // add json type } - // json key encode by binary - void *tagKey = taosMemoryCalloc(keyLen + VARSTR_HEADER_SIZE, 1); - if(!tagKey) { - retCode = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto end; - } - strncpy(varDataVal(tagKey), jsonKey, keyLen); taosHashPut(keyHash, jsonKey, keyLen, &keyLen, CHAR_BYTES); // add key to hash to remove dumplicate, value is useless - varDataSetLen(tagKey, keyLen); - tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKey, varDataTLen(tagKey)); // add json key - taosMemoryFree(tagKey); - if(item->type == cJSON_String){ // add json value format: type|data char *jsonValue = item->valuestring; int32_t valLen = (int32_t)strlen(jsonValue); - char *tagVal = taosMemoryCalloc(valLen * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE + CHAR_BYTES, 1); - if(!tagVal) { + int32_t totalLen = keyLen + VARSTR_HEADER_SIZE + valLen * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE + CHAR_BYTES; + char *tmp = taosMemoryRealloc(tagKV, totalLen); + if(!tmp) { retCode = TSDB_CODE_TSC_OUT_OF_MEMORY; goto end; - } ; - tagVal[0] = TSDB_DATA_TYPE_NCHAR; - char* tagData = POINTER_SHIFT(tagVal, CHAR_BYTES); - if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)varDataVal(tagData), + } + tagKV = tmp; + char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE); + char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES); + *valueType = TSDB_DATA_TYPE_NCHAR; + if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)varDataVal(valueData), (int32_t)(valLen * TSDB_NCHAR_SIZE), &valLen)) { qError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, jsonValue, strerror(errno)); retCode = buildSyntaxErrMsg(pMsgBuf, "charset convert json error", jsonValue); - taosMemoryFree(tagVal); goto end; } - varDataSetLen(tagData, valLen); - tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagVal, CHAR_BYTES + varDataTLen(tagData)); - taosMemoryFree(tagVal); + varDataSetLen(valueData, valLen); + tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, totalLen); }else if(item->type == cJSON_Number){ if(!isfinite(item->valuedouble)){ qError("json value is invalidate"); retCode = buildSyntaxErrMsg(pMsgBuf, "json value number is illegal", json); goto end; } - char tagVal[LONG_BYTES + CHAR_BYTES] = {0}; - tagVal[0] = (item->valuedouble - (int64_t)(item->valuedouble) == 0) ? TSDB_DATA_TYPE_BIGINT - : TSDB_DATA_TYPE_DOUBLE; - char* tagData = POINTER_SHIFT(tagVal,CHAR_BYTES); - if(tagVal[0]== TSDB_DATA_TYPE_DOUBLE) *((double *)tagData) = item->valuedouble; - else if(tagVal[0] == TSDB_DATA_TYPE_BIGINT) *((int64_t *)tagData) = item->valueint; - tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagVal, LONG_BYTES + CHAR_BYTES); + char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE); + char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES); + *valueType = (item->valuedouble - (int64_t)(item->valuedouble) == 0) ? TSDB_DATA_TYPE_BIGINT : TSDB_DATA_TYPE_DOUBLE; + if(*valueType== TSDB_DATA_TYPE_DOUBLE) *((double *)valueData) = item->valuedouble; + else if(*valueType == TSDB_DATA_TYPE_BIGINT) *((int64_t *)valueData) = item->valueint; + tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES +LONG_BYTES); }else if(item->type == cJSON_True || item->type == cJSON_False){ - char tagVal[CHAR_BYTES + CHAR_BYTES] = {0}; - tagVal[0] = TSDB_DATA_TYPE_BOOL; - tagVal[1] = (char)(item->valueint); - tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagVal, CHAR_BYTES + CHAR_BYTES); + char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE); + char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES); + *valueType = TSDB_DATA_TYPE_BOOL; + *valueData = (char)(item->valueint); + tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES + CHAR_BYTES); }else if(item->type == cJSON_NULL){ - char tagVal[CHAR_BYTES] = {TSDB_DATA_TYPE_NULL}; - tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagVal, CHAR_BYTES); + char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE); + *valueType = TSDB_DATA_TYPE_NULL; + tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES); } else{ retCode = buildSyntaxErrMsg(pMsgBuf, "invalidate json value", json); @@ -364,10 +361,11 @@ int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* p } if(taosHashGetSize(keyHash) == 0){ // set json NULL true - tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNULL, CHAR_BYTES); + tdAddColToKVRow(kvRowBuilder, jsonIndex, &jsonNULL, CHAR_BYTES); } end: + taosMemoryFree(tagKV); taosHashCleanup(keyHash); cJSON_Delete(root); return retCode; diff --git a/source/libs/scalar/inc/sclvector.h b/source/libs/scalar/inc/sclvector.h index e51115e9c0..47205b7fba 100644 --- a/source/libs/scalar/inc/sclvector.h +++ b/source/libs/scalar/inc/sclvector.h @@ -53,6 +53,8 @@ static FORCE_INLINE double getVectorDoubleValue_DOUBLE(void *src, int32_t index) return (double)*((double *)src + index); } +double getVectorDoubleValue_JSON(void *src, int32_t index); + static FORCE_INLINE _getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType) { _getDoubleValue_fn_t p = NULL; if (srcType == TSDB_DATA_TYPE_TINYINT) { @@ -77,6 +79,8 @@ static FORCE_INLINE _getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType) p = getVectorDoubleValue_DOUBLE; } else if (srcType == TSDB_DATA_TYPE_TIMESTAMP) { p = getVectorDoubleValue_BIGINT; + } else if (srcType == TSDB_DATA_TYPE_JSON) { + p = getVectorDoubleValue_JSON; } else { assert(0); } diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 711ece3b01..ea249005b4 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -170,7 +170,7 @@ __compar_fn_t gDataCompare[] = {compareInt32Val, compareInt8Val, compareInt16Val compareLenPrefixedWStr, compareUint8Val, compareUint16Val, compareUint32Val, compareUint64Val, setChkInBytes1, setChkInBytes2, setChkInBytes4, setChkInBytes8, compareStrRegexCompMatch, compareStrRegexCompNMatch, setChkNotInBytes1, setChkNotInBytes2, setChkNotInBytes4, setChkNotInBytes8, - compareChkNotInString, compareStrPatternNotMatch, compareWStrPatternNotMatch + compareChkNotInString, compareStrPatternNotMatch, compareWStrPatternNotMatch, compareJsonContainsKey }; int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { @@ -221,7 +221,12 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { assert(0); } } - + + if (optr == OP_TYPE_JSON_CONTAINS && type == TSDB_DATA_TYPE_JSON) { + return 28; + } + + switch (type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: comparFn = 1; break; diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 116dfdd5d5..371b011c40 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -600,44 +600,39 @@ EDealRes sclWalkOperator(SNode* pNode, SScalarCtx *ctx) { return DEAL_RES_CONTINUE; } -EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) { - STargetNode *target = (STargetNode *)pNode; - - if (target->dataBlockId >= taosArrayGetSize(ctx->pBlockList)) { - sclError("target tupleId is too big, tupleId:%d, dataBlockNum:%d", target->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList)); - ctx->code = TSDB_CODE_QRY_INVALID_INPUT; - return DEAL_RES_ERROR; - } - - SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, target->dataBlockId); - if (target->slotId >= taosArrayGetSize(block->pDataBlock)) { - sclError("target slot not exist, dataBlockId:%d, slotId:%d, dataBlockNum:%d", target->dataBlockId, target->slotId, (int32_t)taosArrayGetSize(block->pDataBlock)); - ctx->code = TSDB_CODE_QRY_INVALID_INPUT; - return DEAL_RES_ERROR; - } - - SColumnInfoData *col = taosArrayGet(block->pDataBlock, target->slotId); - - SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES); - if (NULL == res) { - sclError("no valid res in hash, node:%p, type:%d", target->pExpr, nodeType(target->pExpr)); - ctx->code = TSDB_CODE_QRY_APP_ERROR; - return DEAL_RES_ERROR; - } - - for (int32_t i = 0; i < res->numOfRows; ++i) { - if (colDataIsNull(res->columnData, res->numOfRows, i, NULL)) { - colDataAppend(col, i, NULL, true); - } else { - char *p = colDataGetData(res->columnData, i); - colDataAppend(col, i, p, false); - } - } - - sclFreeParam(res); - taosHashRemove(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES); - return DEAL_RES_CONTINUE; -} +//EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) { +// STargetNode *target = (STargetNode *)pNode; +// +// if (target->dataBlockId >= taosArrayGetSize(ctx->pBlockList)) { +// sclError("target tupleId is too big, tupleId:%d, dataBlockNum:%d", target->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList)); +// ctx->code = TSDB_CODE_QRY_INVALID_INPUT; +// return DEAL_RES_ERROR; +// } +// +// SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, target->dataBlockId); +// if (target->slotId >= taosArrayGetSize(block->pDataBlock)) { +// sclError("target slot not exist, dataBlockId:%d, slotId:%d, dataBlockNum:%d", target->dataBlockId, target->slotId, (int32_t)taosArrayGetSize(block->pDataBlock)); +// ctx->code = TSDB_CODE_QRY_INVALID_INPUT; +// return DEAL_RES_ERROR; +// } +// +// // if block->pDataBlock is not enough, there are problems if target->slotId bigger than the size of block->pDataBlock, +// SColumnInfoData *col = taosArrayGet(block->pDataBlock, target->slotId); +// +// SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES); +// if (NULL == res) { +// sclError("no valid res in hash, node:%p, type:%d", target->pExpr, nodeType(target->pExpr)); +// ctx->code = TSDB_CODE_QRY_APP_ERROR; +// return DEAL_RES_ERROR; +// } +// +// colDataAssign(col, res->columnData, res->numOfRows); +// block->info.rows = res->numOfRows; +// +// sclFreeParam(res); +// taosHashRemove(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES); +// return DEAL_RES_CONTINUE; +//} EDealRes sclCalcWalker(SNode* pNode, void* pContext) { if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)) { @@ -657,9 +652,9 @@ EDealRes sclCalcWalker(SNode* pNode, void* pContext) { return sclWalkOperator(pNode, ctx); } - if (QUERY_NODE_TARGET == nodeType(pNode)) { - return sclWalkTarget(pNode, ctx); - } +// if (QUERY_NODE_TARGET == nodeType(pNode)) { +// return sclWalkTarget(pNode, ctx); +// } sclError("invalid node type for scalar calculating, type:%d", nodeType(pNode)); ctx->code = TSDB_CODE_QRY_INVALID_INPUT; diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 05456790a5..11dc60dab3 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -177,24 +177,24 @@ static FORCE_INLINE void varToBool(char *buf, SScalarParam* pOut, int32_t rowInd colDataAppendInt8(pOut->columnData, rowIndex, (int8_t*) &v); } -static FORCE_INLINE void varToNchar(char* buf, SScalarParam* pOut, int32_t rowIndex) { - int32_t len = 0; - int32_t inputLen = varDataLen(buf); - - char* t = taosMemoryCalloc(1,(inputLen + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); - /*int32_t resLen = */taosMbsToUcs4(varDataVal(buf), inputLen, (TdUcs4*) varDataVal(t), pOut->columnData->info.bytes, &len); - varDataSetLen(t, len); - - colDataAppend(pOut->columnData, rowIndex, t, false); - taosMemoryFree(t); -} +//static FORCE_INLINE void varToNchar(char* buf, SScalarParam* pOut, int32_t rowIndex) { +// int32_t len = 0; +// int32_t inputLen = varDataLen(buf); +// +// char* t = taosMemoryCalloc(1,(inputLen + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); +// /*int32_t resLen = */taosMbsToUcs4(varDataVal(buf), inputLen, (TdUcs4*) varDataVal(t), pOut->columnData->info.bytes, &len); +// varDataSetLen(t, len); +// +// colDataAppend(pOut->columnData, rowIndex, t, false); +// taosMemoryFree(t); +//} //TODO opt performance, tmp is not needed. int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, int32_t inType, int32_t outType) { int32_t bufSize = pIn->columnData->info.bytes; char *tmp = taosMemoryMalloc(bufSize + VARSTR_HEADER_SIZE); - bool vton = false; +// bool vton = false; _bufConverteFunc func = NULL; if (TSDB_DATA_TYPE_BOOL == outType) { @@ -205,9 +205,9 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in func = varToUnsigned; } else if (IS_FLOAT_TYPE(outType)) { func = varToFloat; - } else if (outType == TSDB_DATA_TYPE_NCHAR) { - func = varToNchar; - vton = true; +// } else if (outType == TSDB_DATA_TYPE_NCHAR) { // can not be nchar or binary +// func = varToNchar; +// vton = true; } else { sclError("invalid convert outType:%d", outType); return TSDB_CODE_QRY_APP_ERROR; @@ -221,9 +221,9 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in } char* data = colDataGetData(pIn->columnData, i); - if (vton) { - memcpy(tmp, data, varDataTLen(data)); - } else { +// if (vton) { +// memcpy(tmp, data, varDataTLen(data)); +// } else { if (TSDB_DATA_TYPE_VARCHAR == inType) { memcpy(tmp, varDataVal(data), varDataLen(data)); tmp[varDataLen(data)] = 0; @@ -239,7 +239,7 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in tmp[len] = 0; } - } +// } (*func)(tmp, pOut, i); } @@ -248,6 +248,135 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in return TSDB_CODE_SUCCESS; } +void convertNumberToNumber(const void *inData, void *outData, int8_t inType, int8_t outType){ + switch (outType) { + case TSDB_DATA_TYPE_BOOL: { + GET_TYPED_DATA(*((bool *)outData), bool, inType, inData); + break; + } + case TSDB_DATA_TYPE_TINYINT: { + GET_TYPED_DATA(*((int8_t *)outData), int8_t, inType, inData); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + GET_TYPED_DATA(*((int16_t *)outData), int16_t, inType, inData); + break; + } + case TSDB_DATA_TYPE_INT: { + GET_TYPED_DATA(*((int32_t *)outData), int32_t, inType, inData); + break; + } + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: { + GET_TYPED_DATA(*((int64_t *)outData), int64_t, inType, inData); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + GET_TYPED_DATA(*((uint8_t *)outData), uint8_t, inType, inData); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + GET_TYPED_DATA(*((uint16_t *)outData), uint16_t, inType, inData); + break; + } + case TSDB_DATA_TYPE_UINT: { + GET_TYPED_DATA(*((uint32_t *)outData), uint32_t, inType, inData); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + GET_TYPED_DATA(*((uint64_t *)outData), uint64_t, inType, inData); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + GET_TYPED_DATA(*((float *)outData), float, inType, inData); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + GET_TYPED_DATA(*((double *)outData), double, inType, inData); + break; + } + default:{ + ASSERT(0); + } + } +} + +void convertStringToDouble(const void *inData, void *outData, int8_t inType, int8_t outType){ + char *tmp = taosMemoryMalloc(varDataTLen(inData)); + if (inType == TSDB_DATA_TYPE_NCHAR) { + int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(inData), varDataLen(inData), tmp); + if (len < 0) { + sclError("castConvert taosUcs4ToMbs error 1"); + } + + tmp[len] = 0; + } else { + memcpy(tmp, varDataVal(inData), varDataLen(inData)); + tmp[varDataLen(inData)] = 0; + } + + ASSERT(outType == TSDB_DATA_TYPE_DOUBLE); + double value = strtod(tmp, NULL); + + *((double *)outData) = value; + taosMemoryFreeClear(tmp); +} + +double getVectorDoubleValue_JSON(void *src, int32_t index){ + ASSERT(!colDataIsNull_s(((SColumnInfoData*)src), index)); + char *data = colDataGetData((SColumnInfoData*)src, index); + double out = 0; + if(*data == TSDB_DATA_TYPE_NCHAR) { + convertStringToDouble(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE); + } { + convertNumberToNumber(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE); + } + return out; +} + +void convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t typeRight, char **pLeftData, char **pRightData, void *pLeftOut, void *pRightOut){ + if(optr == OP_TYPE_JSON_CONTAINS) { + return; + } + + if(typeLeft != TSDB_DATA_TYPE_JSON && typeRight != TSDB_DATA_TYPE_JSON){ + return; + } + + if(typeLeft == TSDB_DATA_TYPE_JSON){ + typeLeft = **pLeftData; + (*pLeftData) ++; + } + if(typeRight == TSDB_DATA_TYPE_JSON){ + typeRight = **pRightData; + (*pRightData) ++; + } + int8_t type = vectorGetConvertType(typeLeft, typeRight); + + if(type == 0) { + *fp = filterGetCompFunc(typeLeft, optr); + return; + } + + *fp = filterGetCompFunc(type, optr); + + if(typeLeft == TSDB_DATA_TYPE_NCHAR) { + convertStringToDouble(*pLeftData, pLeftOut, typeLeft, type); + *pLeftData = pLeftOut; + } else if(typeLeft != type) { + convertNumberToNumber(*pLeftData, pLeftOut, typeLeft, type); + *pLeftData = pLeftOut; + } + + if(IS_VAR_DATA_TYPE(typeRight)) { + convertStringToDouble(*pRightData, pRightOut, typeRight, type); + *pRightData = pRightOut; + } else if(typeRight != type) { + convertNumberToNumber(*pRightData, pRightOut, typeRight, type); + *pRightData = pRightOut; + } +} + // TODO opt performance int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut) { SColumnInfoData* pInputCol = pIn->columnData; @@ -522,7 +651,7 @@ enum { static int32_t doConvertHelper(SScalarParam* pDest, int32_t* convert, const SScalarParam* pParam, int32_t type) { SColumnInfoData* pCol = pParam->columnData; - if (IS_VAR_DATA_TYPE(pCol->info.type)) { + if (IS_VAR_DATA_TYPE(pCol->info.type) || pCol->info.type != TSDB_DATA_TYPE_JSON) { pDest->numOfRows = pParam->numOfRows; SDataType t = {.type = type, .bytes = tDataTypes[type].bytes}; @@ -556,7 +685,8 @@ static void vectorMathAddHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRig colDataAppendNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { - *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) + getVectorDoubleValueFnRight(pRightCol->pData, 0); + *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i) + + getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), 0); } pOutputCol->hasNull = pLeftCol->hasNull; if (pOutputCol->hasNull) { @@ -587,6 +717,36 @@ static void doReleaseVec(SColumnInfoData* pCol, int32_t type) { } } +char *getJsonValue(char *json, char *key){ //todo + return NULL; +} + +void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + SColumnInfoData *pOutputCol = pOut->columnData; + + int32_t i = ((_ord) == TSDB_ORDER_ASC)? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC)? 1 : -1; + + pOut->numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows); + + char *pRightData = colDataGetData(pRight->columnData, 0); + for (; i >= 0 && i < pLeft->numOfRows; i += step) { + if (pLeft->columnData->varmeta.offset[i] == -1) { + pOutputCol->varmeta.offset[i] = -1; + pOutputCol->hasNull = true; + continue; + } + char *pLeftData = colDataGetData(pLeft->columnData, i); + char *value = getJsonValue(pLeftData, pRightData); + if (!value || *value == TSDB_DATA_TYPE_NULL) { + pOutputCol->varmeta.offset[i] = -1; + pOutputCol->hasNull = true; + continue; + } + colDataAppend(pOutputCol, i, pLeftData, false); + } +} + void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { SColumnInfoData *pOutputCol = pOut->columnData; @@ -605,7 +765,12 @@ void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut double *output = (double *)pOutputCol->pData; if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) + getVectorDoubleValueFnRight(pRightCol->pData, i); + if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore + } + *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i) + + getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i); } pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull); @@ -637,7 +802,8 @@ static void vectorMathSubHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRig colDataAppendNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { - *output = (getVectorDoubleValueFnLeft(pLeftCol->pData, i) - getVectorDoubleValueFnRight(pRightCol->pData, 0)) * factor; + *output = (getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i) + - getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), 0)) * factor; } pOutputCol->hasNull = pLeftCol->hasNull; if (pOutputCol->hasNull) { @@ -664,7 +830,12 @@ void vectorMathSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut double *output = (double *)pOutputCol->pData; if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) - getVectorDoubleValueFnRight(pRightCol->pData, i); + if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore + } + *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i) + - getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i); } pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull); @@ -696,7 +867,8 @@ static void vectorMathMultiplyHelper(SColumnInfoData* pLeftCol, SColumnInfoData* colDataAppendNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { - *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) * getVectorDoubleValueFnRight(pRightCol->pData, 0); + *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i) + * getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), 0); } pOutputCol->hasNull = pLeftCol->hasNull; if (pOutputCol->hasNull) { @@ -722,7 +894,12 @@ void vectorMathMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam double *output = (double *)pOutputCol->pData; if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) * getVectorDoubleValueFnRight(pRightCol->pData, i); + if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore + } + *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i) + * getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i); } pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull); @@ -760,7 +937,12 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p double *output = (double *)pOutputCol->pData; if (pLeft->numOfRows == pRight->numOfRows) { // check for the 0 value for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) / getVectorDoubleValueFnRight(pRightCol->pData, i); + if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore + } + *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i) + /getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i); } pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull); @@ -776,7 +958,8 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows); } else { for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) { - *output = getVectorDoubleValueFnLeft(pLeftCol->pData, 0) / getVectorDoubleValueFnRight(pRightCol->pData, i); + *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), 0) + / getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i); } pOutputCol->hasNull = pRightCol->hasNull; if (pOutputCol->hasNull) { @@ -788,7 +971,8 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows); } else { for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) { - *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) / getVectorDoubleValueFnRight(pRightCol->pData, 0); + *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i) + / getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), 0); } pOutputCol->hasNull = pLeftCol->hasNull; if (pOutputCol->hasNull) { @@ -825,8 +1009,8 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam continue; } - double lx = getVectorDoubleValueFnLeft(pLeftCol->pData, i); - double rx = getVectorDoubleValueFnRight(pRightCol->pData, i); + double lx = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i); + double rx = getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i); if (isnan(lx) || isinf(lx) || isnan(rx) || isinf(rx)) { colDataAppendNULL(pOutputCol, i); continue; @@ -835,7 +1019,7 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *output = lx - ((int64_t)(lx / rx)) * rx; } } else if (pLeft->numOfRows == 1) { - double lx = getVectorDoubleValueFnLeft(pLeftCol->pData, 0); + double lx = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), 0); if (colDataIsNull_f(pLeftCol->nullbitmap, 0) || isnan(lx) || isinf(lx)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows); } else { @@ -845,7 +1029,7 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam continue; } - double rx = getVectorDoubleValueFnRight(pRightCol->pData, i); + double rx = getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i); if (isnan(rx) || isinf(rx) || FLT_EQUAL(rx, 0)) { colDataAppendNULL(pOutputCol, i); continue; @@ -855,7 +1039,7 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam } } } else if (pRight->numOfRows == 1) { - double rx = getVectorDoubleValueFnRight(pRightCol->pData, 0); + double rx = getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), 0); if (colDataIsNull_f(pRightCol->nullbitmap, 0) || FLT_EQUAL(rx, 0)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows); } else { @@ -865,7 +1049,7 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam continue; } - double lx = getVectorDoubleValueFnLeft(pLeftCol->pData, i); + double lx = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i); if (isnan(lx) || isinf(lx)) { colDataAppendNULL(pOutputCol, i); continue; @@ -895,7 +1079,11 @@ void vectorMathMinus(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO double *output = (double *)pOutputCol->pData; for (; i < pLeft->numOfRows && i >= 0; i += step, output += 1) { - *output = - getVectorDoubleValueFnLeft(pLeftCol->pData, i); + if (colDataIsNull_s(pLeft->columnData, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore + } + *output = - getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i); } pOutputCol->hasNull = pLeftCol->hasNull; @@ -1099,30 +1287,39 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * char *pLeftData = colDataGetData(pLeft->columnData, i); char *pRightData = colDataGetData(pRight->columnData, i); + + int64_t leftOut = 0; + int64_t rightOut = 0; + convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut); bool res = filterDoCompare(fp, optr, pLeftData, pRightData); colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); } } else if (pRight->numOfRows == 1) { - char *pRightData = colDataGetData(pRight->columnData, 0); ASSERT(pLeft->pHashFilter == NULL); - for (; i >= 0 && i < pLeft->numOfRows; i += step) { if (colDataIsNull_s(pLeft->columnData, i)) { continue; } char *pLeftData = colDataGetData(pLeft->columnData, i); + char *pRightData = colDataGetData(pRight->columnData, 0); + int64_t leftOut = 0; + int64_t rightOut = 0; + convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut); bool res = filterDoCompare(fp, optr, pLeftData, pRightData); colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); } } else if (pLeft->numOfRows == 1) { - char *pLeftData = colDataGetData(pLeft->columnData, 0); for (; i >= 0 && i < pRight->numOfRows; i += step) { if (colDataIsNull_s(pRight->columnData, i)) { continue; } + char *pLeftData = colDataGetData(pLeft->columnData, 0); char *pRightData = colDataGetData(pLeft->columnData, i); + int64_t leftOut = 0; + int64_t rightOut = 0; + convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut); bool res = filterDoCompare(fp, optr, pLeftData, pRightData); colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); } @@ -1203,6 +1400,10 @@ void vectorNotMatch(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOu vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_NMATCH); } +void vectorJsonContains(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_JSON_CONTAINS); +} + void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { for(int32_t i = 0; i < pLeft->numOfRows; ++i) { int8_t v = colDataIsNull_s(pLeft->columnData, i)? 1:0; @@ -1271,6 +1472,10 @@ _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) { return vectorBitOr; case OP_TYPE_IS_TRUE: return vectorIsTrue; + case OP_TYPE_JSON_GET_VALUE: + return vectorJsonArrow; + case OP_TYPE_JSON_CONTAINS: + return vectorJsonContains; default: assert(0); return NULL; diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index 93022de021..0ddc76e415 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -222,6 +222,11 @@ int32_t compareLenPrefixedWStrDesc(const void *pLeft, const void *pRight) { return compareLenPrefixedWStr(pRight, pLeft); } +int32_t compareJsonContainsKey(const void* pLeft, const void* pRight) { + if(pLeft) return 0; + return 1; +} + /* * Compare two strings * TSDB_MATCH: Match From c8fe5bc88b18278849e98ec46da4a205208e2280 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 18 Apr 2022 10:19:15 +0800 Subject: [PATCH 09/47] stmt --- include/client/taos.h | 36 +--- include/libs/parser/parser.h | 14 ++ source/client/inc/clientInt.h | 6 +- source/client/inc/clientStmt.h | 10 +- source/client/src/clientMain.c | 24 ++- source/client/src/clientStmt.c | 159 +++++++----------- source/client/src/tmq.c | 2 +- source/libs/parser/inc/parInsertData.h | 3 +- source/libs/parser/inc/parInt.h | 16 -- source/libs/parser/src/parInsert.c | 219 +++++++++++++++++++++++-- source/libs/parser/src/parInsertData.c | 189 ++++++--------------- 11 files changed, 354 insertions(+), 324 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index d3856d432e..ac7a671edc 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -92,38 +92,14 @@ typedef struct taosField { typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code); -typedef struct TAOS_BIND { - int buffer_type; - void *buffer; - uintptr_t buffer_length; // unused - uintptr_t *length; - int *is_null; - - int is_unsigned; // unused - int *error; // unused - union { - int64_t ts; - int8_t b; - int8_t v1; - int16_t v2; - int32_t v4; - int64_t v8; - float f4; - double f8; - unsigned char *bin; - char *nchar; - } u; - unsigned int allocated; -} TAOS_BIND; - -typedef struct TAOS_MULTI_BIND { +typedef struct TAOS_BIND_v2 { int buffer_type; void *buffer; uintptr_t buffer_length; int32_t *length; char *is_null; int num; -} TAOS_MULTI_BIND; +} TAOS_BIND_v2; typedef enum { SET_CONF_RET_SUCC = 0, @@ -154,16 +130,16 @@ const char *taos_data_type(int type); DLL_EXPORT TAOS_STMT *taos_stmt_init(TAOS *taos); DLL_EXPORT int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length); -DLL_EXPORT int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_BIND *tags); +DLL_EXPORT int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_BIND_v2 *tags); DLL_EXPORT int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name); DLL_EXPORT int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name); DLL_EXPORT int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert); DLL_EXPORT int taos_stmt_num_params(TAOS_STMT *stmt, int *nums); DLL_EXPORT int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes); -DLL_EXPORT int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND *bind); -DLL_EXPORT int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind); -DLL_EXPORT int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx); +DLL_EXPORT int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND_v2 *bind); +DLL_EXPORT int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind); +DLL_EXPORT int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int colIdx); DLL_EXPORT int taos_stmt_add_batch(TAOS_STMT *stmt); DLL_EXPORT int taos_stmt_execute(TAOS_STMT *stmt); DLL_EXPORT TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt); diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 640b4181cd..30e9b199f1 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -80,6 +80,20 @@ void qDestroyQuery(SQuery* pQueryNode); int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema); +int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash); +void qResetStmtDataBlock(void* pBlock, bool freeData); +int32_t qCloneStmtDataBlock(void** pDst, void* pSrc); +void qFreeStmtDataBlock(void* pDataBlock); +int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc); +void qDestroyStmtDataBlock(void* pBlock); +int32_t qBindStmtColsValue(void *pDataBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen); +int32_t qBuildStmtColFields(void *pDataBlock, int32_t *fieldNum, TAOS_FIELD** fields); +int32_t qBuildStmtTagFields(void *pBlock, void *boundTags, int32_t *fieldNum, TAOS_FIELD** fields); +int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, SName *pName, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen); +void destroyBoundColumnInfo(void* pBoundInfo); +int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char *msgBuf, int32_t msgBufLen); + + #ifdef __cplusplus } #endif diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index b352b65bea..e2a89b8e42 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -280,7 +280,8 @@ void initMsgHandleFp(); TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db, uint16_t port); -int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery); +int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtCallback* pStmtCb); + int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList); int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest); @@ -311,6 +312,9 @@ int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* v // --- mq void hbMgrInitMqHbRspHandle(); +SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery); + + #ifdef __cplusplus } #endif diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index d84bbaeba1..4d7d578f70 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -19,6 +19,9 @@ #ifdef __cplusplus extern "C" { #endif +#include "catalog.h" + +typedef void STableDataBlocks; typedef enum { STMT_TYPE_INSERT = 1, @@ -107,15 +110,16 @@ typedef struct STscStmt { TAOS_STMT *stmtInit(TAOS *taos); int stmtClose(TAOS_STMT *stmt); int stmtExec(TAOS_STMT *stmt); -char *stmtErrstr(TAOS_STMT *stmt); +const char *stmtErrstr(TAOS_STMT *stmt); int stmtAffectedRows(TAOS_STMT *stmt); int stmtPrepare(TAOS_STMT *stmt, const char *sql, unsigned long length); -int stmtSetTbNameTags(TAOS_STMT *stmt, const char *name, TAOS_BIND *tags); +int stmtSetTbName(TAOS_STMT *stmt, const char *tbName); +int stmtSetTbTags(TAOS_STMT *stmt, TAOS_BIND_v2 *tags); int stmtIsInsert(TAOS_STMT *stmt, int *insert); int stmtGetParamNum(TAOS_STMT *stmt, int *nums); int stmtAddBatch(TAOS_STMT *stmt); TAOS_RES *stmtUseResult(TAOS_STMT *stmt); -int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind); +int stmtBindBatch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind); #ifdef __cplusplus diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 91607b5fef..5d1065b59b 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -579,7 +579,7 @@ int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length) { return stmtPrepare(stmt, sql, length); } -int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_BIND *tags) { +int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_BIND_v2 *tags) { if (stmt == NULL || name == NULL) { tscError("NULL parameter for %s", __FUNCTION__); terrno = TSDB_CODE_INVALID_PARA; @@ -608,25 +608,23 @@ int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name) { return stmtSetTbName(stmt, name); } -int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND *bind) { +int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) { if (stmt == NULL || bind == NULL) { tscError("NULL parameter for %s", __FUNCTION__); terrno = TSDB_CODE_INVALID_PARA; return terrno; } - TAOS_MULTI_BIND mbind = {0}; - mbind.buffer_type = bind->buffer_type; - mbind.buffer = bind->buffer; - mbind.buffer_length = bind->buffer_length; - mbind.length = bind->length; - mbind.is_null = bind->is_null; - mbind.num = 1; + if (bind->num > 1) { + tscError("invalid bind number %d for %s", bind->num, __FUNCTION__); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } - return stmtBindBatch(stmt, &mbind); + return stmtBindBatch(stmt, bind); } -int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { +int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) { if (stmt == NULL || bind == NULL) { tscError("NULL parameter for %s", __FUNCTION__); terrno = TSDB_CODE_INVALID_PARA; @@ -642,7 +640,7 @@ int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { return stmtBindBatch(stmt, bind); } -int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx) { +int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int colIdx) { return stmtBindBatch(stmt, bind); /* TODO */ } @@ -703,7 +701,7 @@ char *taos_stmt_errstr(TAOS_STMT *stmt) { return NULL; } - return stmtErrstr(stmt); + return (char *)stmtErrstr(stmt); } int taos_stmt_affected_rows(TAOS_STMT *stmt) { diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 2c0085a600..5a3f232da7 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -48,6 +48,42 @@ int32_t stmtGetExecInfo(TAOS_STMT* stmt, SHashObj** pVgHash, SHashObj** pBlockHa return TSDB_CODE_SUCCESS; } +int32_t stmtCacheBlock(STscStmt *pStmt) { + if (pStmt->sql.type != STMT_TYPE_MULTI_INSERT) { + return TSDB_CODE_SUCCESS; + } + + uint64_t uid; + if (TSDB_CHILD_TABLE == pStmt->bind.tbType) { + uid = pStmt->bind.tbSuid; + } else { + ASSERT(TSDB_NORMAL_TABLE == pStmt->bind.tbType); + uid = pStmt->bind.tbUid; + } + + if (taosHashGet(pStmt->sql.pTableCache, &uid, sizeof(uid))) { + return TSDB_CODE_SUCCESS; + } + + STableDataBlocks** pSrc = taosHashGet(pStmt->exec.pBlockHash, &uid, sizeof(uid)); + STableDataBlocks* pDst = NULL; + + STMT_ERR_RET(qCloneStmtDataBlock(&pDst, *pSrc)); + + SStmtTableCache cache = { + .pDataBlock = pDst, + .boundTags = pStmt->bind.boundTags, + }; + + if (taosHashPut(pStmt->sql.pTableCache, &uid, sizeof(uid), &cache, sizeof(cache))) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pStmt->bind.boundTags = NULL; + + return TSDB_CODE_SUCCESS; +} + int32_t stmtParseSql(STscStmt* pStmt) { SStmtCallback stmtCb = { .pStmt = pStmt, @@ -79,81 +115,6 @@ int32_t stmtParseSql(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } - -void stmtResetDataBlock(STableDataBlocks* pBlock) { - pBlock->pData = NULL; - pBlock->ordered = true; - pBlock->prevTS = INT64_MIN; - pBlock->size = sizeof(SSubmitBlk); - pBlock->tsSource = -1; - pBlock->numOfTables = 1; - pBlock->nAllocSize = TSDB_PAYLOAD_SIZE; - pBlock->headerSize = pBlock->size; - - memset(&pBlock->rowBuilder, 0, sizeof(pBlock->rowBuilder)); -} - - -int32_t stmtCloneDataBlock(STableDataBlocks** pDst, STableDataBlocks* pSrc) { - *pDst = (STableDataBlocks*)taosMemoryMalloc(sizeof(STableDataBlocks)); - if (NULL == *pDst) { - STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); - } - - memcpy(*pDst, pSrc, sizeof(STableDataBlocks)); - (*pDst)->cloned = true; - - stmtResetDataBlock(*pDst); - - return TSDB_CODE_SUCCESS; -} - -void stmtFreeDataBlock(STableDataBlocks* pDataBlock) { - if (pDataBlock == NULL) { - return; - } - - taosMemoryFreeClear(pDataBlock->pData); - taosMemoryFreeClear(pDataBlock); -} - - -int32_t stmtCacheBlock(STscStmt *pStmt) { - if (pStmt->sql.type != STMT_TYPE_MULTI_INSERT) { - return TSDB_CODE_SUCCESS; - } - - uint64_t uid; - if (TSDB_CHILD_TABLE == pStmt->bind.tbType) { - uid = pStmt->bind.tbSuid; - } else { - ASSERT(TSDB_NORMAL_TABLE == pStmt->bind.tbType); - uid = pStmt->bind.tbUid; - } - - if (taosHashGet(pStmt->sql.pTableCache, &uid, sizeof(uid))) { - return TSDB_CODE_SUCCESS; - } - - STableDataBlocks** pSrc = taosHashGet(pStmt->exec.pBlockHash, &uid, sizeof(uid)); - STableDataBlocks* pDst = NULL; - - STMT_ERR_RET(stmtCloneDataBlock(&pDst, *pSrc)); - - SStmtTableCache cache = { - .pDataBlock = pDst, - .boundTags = pStmt->bind.boundTags, - }; - - if (taosHashPut(pStmt->sql.pTableCache, &uid, sizeof(uid), &cache, sizeof(cache))) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - pStmt->bind.boundTags = NULL; - - return TSDB_CODE_SUCCESS; -} - int32_t stmtCleanBindInfo(STscStmt* pStmt) { pStmt->bind.tbUid = 0; pStmt->bind.tbSuid = 0; @@ -163,6 +124,8 @@ int32_t stmtCleanBindInfo(STscStmt* pStmt) { taosMemoryFreeClear(pStmt->bind.tbName); destroyBoundColumnInfo(pStmt->bind.boundTags); taosMemoryFreeClear(pStmt->bind.boundTags); + + return TSDB_CODE_SUCCESS; } int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable) { @@ -172,16 +135,17 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable) { void *pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL); while (pIter) { STableDataBlocks* pBlocks = *(STableDataBlocks**)pIter; - - if (keepTable && (*(uint64_t*)taosHashGetKey(pIter, NULL) == pStmt->bind.tbUid)) { - taosMemoryFreeClear(pBlocks->pData); - stmtResetDataBlock(pBlocks); + uint64_t *key = taosHashGetKey(pIter, NULL); + + if (keepTable && (*key == pStmt->bind.tbUid)) { + qResetStmtDataBlock(pBlocks, true); pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter); continue; } - stmtFreeDataBlock(pBlocks); + qFreeStmtDataBlock(pBlocks); + taosHashRemove(pStmt->exec.pBlockHash, key, sizeof(*key)); pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter); } @@ -206,8 +170,7 @@ int32_t stmtCleanSQLInfo(STscStmt* pStmt) { while (pIter) { SStmtTableCache* pCache = *(SStmtTableCache**)pIter; - pCache->pDataBlock->cloned = false; - destroyDataBlock(pCache->pDataBlock); + qDestroyStmtDataBlock(pCache->pDataBlock); destroyBoundColumnInfo(pCache->boundTags); pIter = taosHashIterate(pStmt->sql.pTableCache, pIter); @@ -269,13 +232,7 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { pStmt->bind.boundTags = pCache->boundTags; STableDataBlocks* pNewBlock = NULL; - STMT_ERR_RET(stmtCloneDataBlock(&pNewBlock, pCache->pDataBlock)); - - pNewBlock->pData = taosMemoryMalloc(pNewBlock->nAllocSize); - if (NULL == pNewBlock->pData) { - stmtFreeDataBlock(pNewBlock); - STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); - } + STMT_ERR_RET(qRebuildStmtDataBlock(&pNewBlock, pCache->pDataBlock)); if (taosHashPut(pStmt->exec.pBlockHash, &pStmt->bind.tbUid, sizeof(pStmt->bind.tbUid), &pNewBlock, POINTER_BYTES)) { STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); @@ -319,7 +276,7 @@ int stmtPrepare(TAOS_STMT *stmt, const char *sql, unsigned long length) { STMT_ERR_RET(stmtCleanSQLInfo(pStmt)); } - STMT_SWITCH_STATUS(stmt, STMT_PREPARE, TSDB_CODE_TSC_STMT_API_ERROR); + STMT_SWITCH_STATUS(pStmt, STMT_PREPARE, TSDB_CODE_TSC_STMT_API_ERROR); pStmt->sql.sqlStr = strndup(sql, length); pStmt->sql.sqlLen = length; @@ -331,7 +288,7 @@ int stmtPrepare(TAOS_STMT *stmt, const char *sql, unsigned long length) { int stmtSetTbName(TAOS_STMT *stmt, const char *tbName) { STscStmt* pStmt = (STscStmt*)stmt; - STMT_SWITCH_STATUS(stmt, STMT_SETTBNAME, TSDB_CODE_TSC_STMT_API_ERROR); + STMT_SWITCH_STATUS(pStmt, STMT_SETTBNAME, TSDB_CODE_TSC_STMT_API_ERROR); taosMemoryFree(pStmt->bind.tbName); @@ -348,10 +305,10 @@ int stmtSetTbName(TAOS_STMT *stmt, const char *tbName) { return TSDB_CODE_SUCCESS; } -int stmtSetTbTags(TAOS_STMT *stmt, TAOS_BIND *tags) { +int stmtSetTbTags(TAOS_STMT *stmt, TAOS_BIND_v2 *tags) { STscStmt* pStmt = (STscStmt*)stmt; - STMT_SWITCH_STATUS(stmt, STMT_SETTBNAME, TSDB_CODE_TSC_STMT_API_ERROR); + STMT_SWITCH_STATUS(pStmt, STMT_SETTBNAME, TSDB_CODE_TSC_STMT_API_ERROR); if (pStmt->bind.needParse) { STMT_ERR_RET(stmtParseSql(pStmt)); @@ -372,7 +329,7 @@ int stmtSetTbTags(TAOS_STMT *stmt, TAOS_BIND *tags) { int32_t stmtFetchTagFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fields) { STscStmt* pStmt = (STscStmt*)stmt; - STMT_SWITCH_STATUS(stmt, STMT_FETCH_TAG_FIELDS, TSDB_CODE_TSC_STMT_API_ERROR); + STMT_SWITCH_STATUS(pStmt, STMT_FETCH_TAG_FIELDS, TSDB_CODE_TSC_STMT_API_ERROR); if (pStmt->bind.needParse) { STMT_ERR_RET(stmtParseSql(pStmt)); @@ -394,10 +351,10 @@ int32_t stmtFetchTagFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fiel return TSDB_CODE_SUCCESS; } -int32_t stmtFetchColFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD* fields) { +int32_t stmtFetchColFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fields) { STscStmt* pStmt = (STscStmt*)stmt; - STMT_SWITCH_STATUS(stmt, STMT_FETCH_COL_FIELDS, TSDB_CODE_TSC_STMT_API_ERROR); + STMT_SWITCH_STATUS(pStmt, STMT_FETCH_COL_FIELDS, TSDB_CODE_TSC_STMT_API_ERROR); if (pStmt->bind.needParse) { STMT_ERR_RET(stmtParseSql(pStmt)); @@ -419,10 +376,10 @@ int32_t stmtFetchColFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD* field return TSDB_CODE_SUCCESS; } -int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { +int stmtBindBatch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) { STscStmt* pStmt = (STscStmt*)stmt; - STMT_SWITCH_STATUS(stmt, STMT_BIND, TSDB_CODE_TSC_STMT_API_ERROR); + STMT_SWITCH_STATUS(pStmt, STMT_BIND, TSDB_CODE_TSC_STMT_API_ERROR); if (pStmt->bind.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { pStmt->bind.needParse = false; @@ -451,7 +408,7 @@ int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { int stmtAddBatch(TAOS_STMT *stmt) { STscStmt* pStmt = (STscStmt*)stmt; - STMT_SWITCH_STATUS(stmt, STMT_ADD_BATCH, TSDB_CODE_TSC_STMT_API_ERROR); + STMT_SWITCH_STATUS(pStmt, STMT_ADD_BATCH, TSDB_CODE_TSC_STMT_API_ERROR); STMT_ERR_RET(stmtCacheBlock(pStmt)); @@ -462,7 +419,7 @@ int stmtExec(TAOS_STMT *stmt) { STscStmt* pStmt = (STscStmt*)stmt; int32_t code = 0; - STMT_SWITCH_STATUS(stmt, STMT_EXECUTE, TSDB_CODE_TSC_STMT_API_ERROR); + STMT_SWITCH_STATUS(pStmt, STMT_EXECUTE, TSDB_CODE_TSC_STMT_API_ERROR); STMT_ERR_RET(qBuildStmtOutput(pStmt->sql.pQuery, pStmt->exec.pVgHash, pStmt->exec.pBlockHash)); @@ -484,7 +441,7 @@ int stmtClose(TAOS_STMT *stmt) { return TSDB_CODE_SUCCESS; } -char *stmtErrstr(TAOS_STMT *stmt) { +const char *stmtErrstr(TAOS_STMT *stmt) { STscStmt* pStmt = (STscStmt*)stmt; if (stmt == NULL) { diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index dbe78782f5..70b87f51c0 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -582,7 +582,7 @@ TAOS_RES* tmq_create_stream(TAOS* taos, const char* streamName, const char* tbNa int32_t code = 0; CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); - CHECK_CODE_GOTO(parseSql(pRequest, false, &pQueryNode), _return); + CHECK_CODE_GOTO(parseSql(pRequest, false, &pQueryNode, NULL), _return); // todo check for invalid sql statement and return with error code diff --git a/source/libs/parser/inc/parInsertData.h b/source/libs/parser/inc/parInsertData.h index f0fffff38b..6b45d16d1a 100644 --- a/source/libs/parser/inc/parInsertData.h +++ b/source/libs/parser/inc/parInsertData.h @@ -131,7 +131,6 @@ static FORCE_INLINE int32_t setBlockInfo(SSubmitBlk *pBlocks, STableDataBlocks* int32_t schemaIdxCompar(const void *lhs, const void *rhs); int32_t boundIdxCompar(const void *lhs, const void *rhs); void setBoundColumnInfo(SParsedDataColInfo *pColList, SSchema *pSchema, col_id_t numOfCols); -void destroyBoundColumnInfo(void* pBoundInfo); void destroyBlockArrayList(SArray* pDataBlockList); void destroyBlockHashmap(SHashObj* pDataBlockHash); int initRowBuilder(SRowBuilder *pBuilder, int16_t schemaVer, SParsedDataColInfo *pColInfo); @@ -139,5 +138,7 @@ int32_t allocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList, SVCreateTbReq* pCreateTbReq); int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** pVgDataBlocks); +int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq); +int32_t allocateMemForSize(STableDataBlocks *pDataBlock, int32_t allSize); #endif // TDENGINE_DATABLOCKMGT_H diff --git a/source/libs/parser/inc/parInt.h b/source/libs/parser/inc/parInt.h index 6b9b90c513..87348e292e 100644 --- a/source/libs/parser/inc/parInt.h +++ b/source/libs/parser/inc/parInt.h @@ -24,27 +24,11 @@ extern "C" { #include "parToken.h" #include "parUtil.h" -typedef struct SKvParam { - SKVRowBuilder *builder; - SSchema *schema; - char buf[TSDB_MAX_TAGS_LEN]; -} SKvParam; - -#define CHECK_CODE(expr) \ - do { \ - int32_t code = expr; \ - if (TSDB_CODE_SUCCESS != code) { \ - return code; \ - } \ - } while (0) - int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery); int32_t parse(SParseContext* pParseCxt, SQuery** pQuery); int32_t translate(SParseContext* pParseCxt, SQuery* pQuery); int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema); int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery); -int32_t createSName(SName* pName, SToken* pTableName, int32_t acctId, const char* dbName, SMsgBuf* pMsgBuf); -int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void *value, int32_t len, void *param); #ifdef __cplusplus } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 4d89f70a59..1917b8372e 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -62,6 +62,29 @@ typedef int32_t (*_row_append_fn_t)(SMsgBuf* pMsgBuf, const void *value, int32_t static uint8_t TRUE_VALUE = (uint8_t)TSDB_TRUE; static uint8_t FALSE_VALUE = (uint8_t)TSDB_FALSE; +typedef struct SKvParam { + SKVRowBuilder *builder; + SSchema *schema; + char buf[TSDB_MAX_TAGS_LEN]; +} SKvParam; + +typedef struct SMemParam { + SRowBuilder* rb; + SSchema* schema; + int32_t toffset; + col_id_t colIdx; +} SMemParam; + + +#define CHECK_CODE(expr) \ + do { \ + int32_t code = expr; \ + if (TSDB_CODE_SUCCESS != code) { \ + return code; \ + } \ + } while (0) + + static int32_t skipInsertInto(SInsertParseContext* pCxt) { SToken sToken; NEXT_TOKEN(pCxt->pSql, sToken); @@ -156,7 +179,7 @@ static int32_t buildName(SInsertParseContext* pCxt, SToken* pStname, char* fullD } -int32_t createSName(SName* pName, SToken* pTableName, int32_t acctId, const char* dbName, SMsgBuf* pMsgBuf) { +static int32_t createSName(SName* pName, SToken* pTableName, int32_t acctId, const char* dbName, SMsgBuf* pMsgBuf) { const char* msg1 = "name too long"; const char* msg2 = "invalid database name"; const char* msg3 = "db is not specified"; @@ -294,7 +317,7 @@ static int32_t buildOutput(SInsertParseContext* pCxt) { return TSDB_CODE_SUCCESS; } -static int32_t checkTimestamp(STableDataBlocks *pDataBlocks, const char *start) { +int32_t checkTimestamp(STableDataBlocks *pDataBlocks, const char *start) { // once the data block is disordered, we do NOT keep previous timestamp any more if (!pDataBlocks->ordered) { return TSDB_CODE_SUCCESS; @@ -600,13 +623,6 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int return TSDB_CODE_FAILED; } -typedef struct SMemParam { - SRowBuilder* rb; - SSchema* schema; - int32_t toffset; - col_id_t colIdx; -} SMemParam; - static FORCE_INLINE int32_t MemRowAppend(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param) { SMemParam* pa = (SMemParam*)param; SRowBuilder* rb = pa->rb; @@ -713,7 +729,7 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* return TSDB_CODE_SUCCESS; } -int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void *value, int32_t len, void *param) { +static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void *value, int32_t len, void *param) { SKvParam* pa = (SKvParam*) param; int8_t type = pa->schema->type; @@ -979,7 +995,7 @@ static int32_t parseValuesClause(SInsertParseContext* pCxt, STableDataBlocks* da return TSDB_CODE_SUCCESS; } -static void destroyCreateSubTbReq(SVCreateTbReq* pReq) { +void destroyCreateSubTbReq(SVCreateTbReq* pReq) { taosMemoryFreeClear(pReq->dbFName); taosMemoryFreeClear(pReq->name); taosMemoryFreeClear(pReq->ctbCfg.pTag); @@ -1111,7 +1127,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } memcpy(tags, &pCxt->tags, sizeof(pCxt->tags)); - (*pCxt->pStmtCb->setBindInfoFn)(pCxt->pTableMeta, tags); + (*pCxt->pStmtCb->setBindInfoFn)(pCxt->pStmtCb->pStmt, pCxt->pTableMeta, tags); memset(&pCxt->tags, 0, sizeof(pCxt->tags)); (*pCxt->pStmtCb->setExecInfoFn)(pCxt->pStmtCb->pStmt, pCxt->pVgroupsHashObj, pCxt->pTableBlockHashObj); @@ -1149,8 +1165,8 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { if (pContext->pStmtCb && *pQuery) { (*pContext->pStmtCb->getExecInfoFn)(pContext->pStmtCb->pStmt, &context.pVgroupsHashObj, &context.pTableBlockHashObj); } else { - context.pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false), - context.pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false), + context.pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); + context.pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); } if (NULL == context.pVgroupsHashObj || NULL == context.pTableBlockHashObj || @@ -1184,7 +1200,7 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { } -int32_t qCreateSName(SName* pName, char* pTableName, int32_t acctId, char* dbName, char *msgBuf, int32_t msgBufLen) { +int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char *msgBuf, int32_t msgBufLen) { SMsgBuf msg = {.buf = msgBuf, .len =msgBufLen}; SToken sToken; int32_t code = 0; @@ -1203,7 +1219,7 @@ int32_t qCreateSName(SName* pName, char* pTableName, int32_t acctId, char* dbNam NEXT_TOKEN(pTableName, sToken); - if (SToken.n > 0) { + if (sToken.n > 0) { return buildInvalidOperationMsg(&msg, "table name format is wrong"); } @@ -1217,12 +1233,12 @@ int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash SInsertParseContext insertCtx = { .pVgroupsHashObj = pVgHash, .pTableBlockHashObj = pBlockHash, - .pOutput = pQuery->pRoot + .pOutput = (SVnodeModifOpStmt*)pQuery->pRoot, }; // merge according to vgId if (taosHashGetSize(insertCtx.pTableBlockHashObj) > 0) { - CHECK_CODE_GOTO(mergeTableDataBlocks(insertCtx.pTableBlockHashObj, modifyNode->payloadType, &insertCtx.pVgDataBlocks), _return); + CHECK_CODE(mergeTableDataBlocks(insertCtx.pTableBlockHashObj, modifyNode->payloadType, &insertCtx.pVgDataBlocks)); } CHECK_CODE(buildOutput(&insertCtx)); @@ -1230,4 +1246,171 @@ int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash return TSDB_CODE_SUCCESS; } +int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, SName *pName, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen){ + STableDataBlocks *pDataBlock = (STableDataBlocks *)pBlock; + SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; + SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; + if (NULL == tags) { + return TSDB_CODE_QRY_APP_ERROR; + } + + SKVRowBuilder tagBuilder; + if (tdInitKVRowBuilder(&tagBuilder) < 0) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SSchema* pSchema = getTableTagSchema(pDataBlock->pTableMeta); + SKvParam param = {.builder = &tagBuilder}; + + for (int c = 0; c < tags->numOfBound; ++c) { + if (bind[c].is_null && bind[c].is_null[0]) { + KvRowAppend(&pBuf, NULL, 0, ¶m); + continue; + } + + SSchema* pTagSchema = &pSchema[tags->boundColumns[c] - 1]; // colId starts with 1 + param.schema = pTagSchema; + + int32_t colLen = pTagSchema->bytes; + if (IS_VAR_DATA_TYPE(pTagSchema->type)) { + colLen = bind[c].length[0]; + } + + CHECK_CODE(KvRowAppend(&pBuf, (char *)bind[c].buffer, colLen, ¶m)); + } + + SKVRow row = tdGetKVRowFromBuilder(&tagBuilder); + if (NULL == row) { + tdDestroyKVRowBuilder(&tagBuilder); + return buildInvalidOperationMsg(&pBuf, "tag value expected"); + } + tdSortKVRowByColIdx(row); + + SVCreateTbReq tbReq = {0}; + CHECK_CODE(buildCreateTbReq(&tbReq, pName, row, suid)); + CHECK_CODE(buildCreateTbMsg(pDataBlock, &tbReq)); + + destroyCreateSubTbReq(&tbReq); + tdDestroyKVRowBuilder(&tagBuilder); + + return TSDB_CODE_SUCCESS; +} + + +int32_t qBindStmtColsValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen) { + STableDataBlocks *pDataBlock = (STableDataBlocks *)pBlock; + SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); + int32_t extendedRowSize = getExtendedRowSize(pDataBlock); + SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; + SRowBuilder* pBuilder = &pDataBlock->rowBuilder; + SMemParam param = {.rb = pBuilder}; + SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; + + CHECK_CODE(allocateMemForSize(pDataBlock, extendedRowSize * bind->num)); + + for (int32_t r = 0; r < bind->num; ++r) { + STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size); // skip the SSubmitBlk header + tdSRowResetBuf(pBuilder, row); + + // 1. set the parsed value from sql string + for (int c = 0; c < spd->numOfBound; ++c) { + SSchema* pColSchema = &pSchema[spd->boundColumns[c] - 1]; + + param.schema = pColSchema; + getSTSRowAppendInfo(pBuilder->rowType, spd, c, ¶m.toffset, ¶m.colIdx); + + if (bind[c].is_null && bind[c].is_null[r]) { + CHECK_CODE(MemRowAppend(&pBuf, NULL, 0, ¶m)); + } else { + int32_t colLen = pColSchema->bytes; + if (IS_VAR_DATA_TYPE(pColSchema->type)) { + colLen = bind[c].length[r]; + } + + CHECK_CODE(MemRowAppend(&pBuf, (char *)bind[c].buffer + bind[c].buffer_length * r, colLen, ¶m)); + } + + if (PRIMARYKEY_TIMESTAMP_COL_ID == pColSchema->colId) { + TSKEY tsKey = TD_ROW_KEY(row); + checkTimestamp(pDataBlock, (const char *)&tsKey); + } + } + + // set the null value for the columns that do not assign values + if ((spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { + for (int32_t i = 0; i < spd->numOfCols; ++i) { + if (spd->cols[i].valStat == VAL_STAT_NONE) { // the primary TS key is not VAL_STAT_NONE + tdAppendColValToTpRow(pBuilder, TD_VTYPE_NONE, getNullValue(pSchema[i].type), true, pSchema[i].type, i, + spd->cols[i].toffset); + } + } + } + + pDataBlock->size += extendedRowSize; + } + + SSubmitBlk *pBlocks = (SSubmitBlk *)(pDataBlock->pData); + if (TSDB_CODE_SUCCESS != setBlockInfo(pBlocks, pDataBlock, bind->num)) { + return buildInvalidOperationMsg(&pBuf, "too many rows in sql, total number of rows should be less than 32767"); + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t buildBoundFields(SParsedDataColInfo *boundInfo, SSchema *pSchema, int32_t *fieldNum, TAOS_FIELD** fields) { + *fields = taosMemoryCalloc(boundInfo->numOfBound, sizeof(TAOS_FIELD)); + if (NULL == *fields) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < boundInfo->numOfBound; ++i) { + SSchema* pTagSchema = &pSchema[boundInfo->boundColumns[i] - 1]; + strcpy((*fields)[i].name, pTagSchema->name); + (*fields)[i].type = pTagSchema->type; + (*fields)[i].bytes = pTagSchema->bytes; + } + + *fieldNum = boundInfo->numOfBound; + + return TSDB_CODE_SUCCESS; +} + + +int32_t qBuildStmtTagFields(void *pBlock, void *boundTags, int32_t *fieldNum, TAOS_FIELD** fields) { + STableDataBlocks *pDataBlock = (STableDataBlocks *)pBlock; + SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; + if (NULL == tags) { + return TSDB_CODE_QRY_APP_ERROR; + } + + SSchema* pSchema = getTableTagSchema(pDataBlock->pTableMeta); + if (tags->numOfBound <= 0) { + *fieldNum = 0; + *fields = NULL; + + return TSDB_CODE_SUCCESS; + } + + CHECK_CODE(buildBoundFields(tags, pSchema, fieldNum, fields)); + + return TSDB_CODE_SUCCESS; +} + +int32_t qBuildStmtColFields(void *pBlock, int32_t *fieldNum, TAOS_FIELD** fields) { + STableDataBlocks *pDataBlock = (STableDataBlocks *)pBlock; + SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); + if (pDataBlock->boundColumnInfo.numOfBound <= 0) { + *fieldNum = 0; + *fields = NULL; + + return TSDB_CODE_SUCCESS; + } + + CHECK_CODE(buildBoundFields(&pDataBlock->boundColumnInfo, pSchema, fieldNum, fields)); + + return TSDB_CODE_SUCCESS; +} + + diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index 9b31924c5e..a7481109fa 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -156,7 +156,7 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star return TSDB_CODE_SUCCESS; } -static int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq) { +int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq) { int32_t len = tSerializeSVCreateTbReq(NULL, pCreateTbReq); if (pBlocks->nAllocSize - pBlocks->size < len) { pBlocks->nAllocSize += len + pBlocks->rowSize; @@ -571,166 +571,75 @@ int initRowBuilder(SRowBuilder *pBuilder, int16_t schemaVer, SParsedDataColInfo return TSDB_CODE_SUCCESS; } -int32_t qBindStmtTagsValue(STableDataBlocks *pDataBlock, void *boundTags, int64_t suid, SName *pName, TAOS_BIND *bind, char *msgBuf, int32_t msgBufLen){ - SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; - SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; - if (NULL == tags) { - return TSDB_CODE_QRY_APP_ERROR; + +void qResetStmtDataBlock(void* block, bool freeData) { + STableDataBlocks* pBlock = (STableDataBlocks*)block; + + if (freeData) { + taosMemoryFree(pBlock->pData); } - SKVRowBuilder tagBuilder; - if (tdInitKVRowBuilder(&tagBuilder) < 0) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + pBlock->pData = NULL; + pBlock->ordered = true; + pBlock->prevTS = INT64_MIN; + pBlock->size = sizeof(SSubmitBlk); + pBlock->tsSource = -1; + pBlock->numOfTables = 1; + pBlock->nAllocSize = TSDB_PAYLOAD_SIZE; + pBlock->headerSize = pBlock->size; + + memset(&pBlock->rowBuilder, 0, sizeof(pBlock->rowBuilder)); +} + + +int32_t qCloneStmtDataBlock(void** pDst, void* pSrc) { + *pDst = taosMemoryMalloc(sizeof(STableDataBlocks)); + if (NULL == *pDst) { + return TSDB_CODE_OUT_OF_MEMORY; } - - SSchema* pSchema = getTableTagSchema(pDataBlock->pTableMeta); - SKvParam param = {.builder = &tagBuilder}; - - for (int c = 0; c < tags->numOfBound; ++c) { - if (bind[c].is_null && bind[c].is_null[0]) { - KvRowAppend(&pBuf, NULL, 0, ¶m); - continue; - } - - SSchema* pTagSchema = &pSchema[tags->boundColumns[c] - 1]; // colId starts with 1 - param.schema = pTagSchema; - - int32_t colLen = pTagSchema->bytes; - if (IS_VAR_DATA_TYPE(pTagSchema->type)) { - colLen = bind[c].length[0]; - } - - CHECK_CODE(KvRowAppend(&pBuf, (char *)bind[c].buffer, colLen, ¶m)); - } - - SKVRow row = tdGetKVRowFromBuilder(&tagBuilder); - if (NULL == row) { - tdDestroyKVRowBuilder(&tagBuilder); - return buildInvalidOperationMsg(&pBuf, "tag value expected"); - } - tdSortKVRowByColIdx(row); - - SVCreateTbReq tbReq = {0}; - CHECK_CODE(buildCreateTbReq(&tbReq, pName, row, suid)); - CHECK_CODE(buildCreateTbMsg(pDataBlock, &tbReq)); - - destroyCreateSubTbReq(&tbReq); - tdDestroyKVRowBuilder(&tagBuilder); + + memcpy(*pDst, pSrc, sizeof(STableDataBlocks)); + ((STableDataBlocks*)(*pDst))->cloned = true; + + qResetStmtDataBlock(*pDst, false); return TSDB_CODE_SUCCESS; } - -int32_t qBindStmtColsValue(STableDataBlocks *pDataBlock, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen) { - SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); - int32_t extendedRowSize = getExtendedRowSize(pDataBlock); - SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; - SRowBuilder* pBuilder = &pDataBlock->rowBuilder; - SMemParam param = {.rb = pBuilder}; - SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; - - CHECK_CODE(allocateMemForSize(pDataBlock, extendedRowSize * bind->num); - - for (int32_t r = 0; r < bind->num; ++r) { - STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size); // skip the SSubmitBlk header - tdSRowResetBuf(pBuilder, row); - - // 1. set the parsed value from sql string - for (int c = 0; c < spd->numOfBound; ++c) { - SSchema* pColSchema = &pSchema[spd->boundColumns[c] - 1]; - - param.schema = pColSchema; - getSTSRowAppendInfo(pBuilder->rowType, spd, c, ¶m.toffset, ¶m.colIdx); - - if (bind[c].is_null && bind[c].is_null[r]) { - CHECK_CODE(MemRowAppend(&pBuf, NULL, 0, ¶m)); - } else { - int32_t colLen = pColSchema->bytes; - if (IS_VAR_DATA_TYPE(pColSchema->type)) { - colLen = bind[c].length[r]; - } - - CHECK_CODE(MemRowAppend(&pBuf, (char *)bind[c].buffer + bind[c].buffer_length * r, colLen, ¶m)); - } - - if (PRIMARYKEY_TIMESTAMP_COL_ID == pColSchema->colId) { - TSKEY tsKey = TD_ROW_KEY(row); - checkTimestamp(pDataBlock, (const char *)&tsKey); - } - } - - // set the null value for the columns that do not assign values - if ((spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { - for (int32_t i = 0; i < spd->numOfCols; ++i) { - if (spd->cols[i].valStat == VAL_STAT_NONE) { // the primary TS key is not VAL_STAT_NONE - tdAppendColValToTpRow(pBuilder, TD_VTYPE_NONE, getNullValue(pSchema[i].type), true, pSchema[i].type, i, - spd->cols[i].toffset); - } - } - } - - pDataBlock->size += extendedRowSize; - } - - SSubmitBlk *pBlocks = (SSubmitBlk *)(pDataBlock->pData); - if (TSDB_CODE_SUCCESS != setBlockInfo(pBlocks, pDataBlock, bind->num)) { - return buildInvalidOperationMsg(&pBuf, "too many rows in sql, total number of rows should be less than 32767"); +int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc) { + int32_t code = qCloneStmtDataBlock(pDst, pSrc); + if (code) { + return code; } - return TSDB_CODE_SUCCESS; -} - - -int32_t buildBoundFields(SParsedDataColInfo *boundInfo, SSchema *pSchema, int32_t *fieldNum, TAOS_FIELD** fields) { - *fields = taosMemoryCalloc(boundInfo->numOfBound, sizeof(TAOS_FIELD)); - if (NULL == *fields) { + STableDataBlocks *pBlock = (STableDataBlocks*)*pDst; + pBlock->pData = taosMemoryMalloc(pBlock->nAllocSize); + if (NULL == pBlock->pData) { + qFreeStmtDataBlock(pBlock); return TSDB_CODE_OUT_OF_MEMORY; } - for (int32_t i = 0; i < boundInfo->numOfBound; ++i) { - SSchema* pTagSchema = &pSchema[boundInfo->boundColumns[i] - 1]; - strcpy((*fields)[i].name, pTagSchema->name); - (*fields)[i].type = pTagSchema->type; - (*fields)[i].bytes = pTagSchema->bytes; - } - - *fieldNum = boundInfo->numOfBound; - return TSDB_CODE_SUCCESS; } -int32_t qBuildStmtTagFields(STableDataBlocks *pDataBlock, void *boundTags, int32_t *fieldNum, TAOS_FIELD** fields) { - SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; - if (NULL == tags) { - return TSDB_CODE_QRY_APP_ERROR; - } - - SSchema* pSchema = getTableTagSchema(pDataBlock->pTableMeta); - if (tags->numOfBound <= 0) { - *fieldNum = 0; - *fields = NULL; - - return TSDB_CODE_SUCCESS; +void qFreeStmtDataBlock(void* pDataBlock) { + if (pDataBlock == NULL) { + return; } - CHECK_CODE(buildBoundFields(tags, pSchema, fieldNum, fields)); - - return TSDB_CODE_SUCCESS; + taosMemoryFreeClear(((STableDataBlocks*)pDataBlock)->pData); + taosMemoryFreeClear(pDataBlock); } -int32_t qBuildStmtColFields(STableDataBlocks *pDataBlock, int32_t *fieldNum, TAOS_FIELD** fields) { - SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); - if (pDataBlock->boundColumnInfo.numOfBound <= 0) { - *fieldNum = 0; - *fields = NULL; - - return TSDB_CODE_SUCCESS; +void qDestroyStmtDataBlock(void* pBlock) { + if (pBlock == NULL) { + return; } - CHECK_CODE(buildBoundFields(&pDataBlock->boundColumnInfo, pSchema, fieldNum, fields)); - - return TSDB_CODE_SUCCESS; + STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; + + pDataBlock->cloned = false; + destroyDataBlock(pDataBlock); } - From 1e1cca985705df0373515a588b995ca3edeb514d Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 18 Apr 2022 13:22:47 +0800 Subject: [PATCH 10/47] stmt --- source/client/inc/clientStmt.h | 7 +- source/client/src/clientMain.c | 4 ++ source/client/src/clientStmt.c | 116 ++++++++++++++++---------------- tests/script/api/batchprepare.c | 85 +++++++++++++---------- 4 files changed, 114 insertions(+), 98 deletions(-) diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index 4d7d578f70..423d474d2e 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -26,7 +26,7 @@ typedef void STableDataBlocks; typedef enum { STMT_TYPE_INSERT = 1, STMT_TYPE_MULTI_INSERT, - STMT_TYPE_QUERY + STMT_TYPE_QUERY, } STMT_TYPE; typedef enum { @@ -38,7 +38,7 @@ typedef enum { STMT_BIND, STMT_BIND_COL, STMT_ADD_BATCH, - STMT_EXECUTE + STMT_EXECUTE, } STMT_STATUS; typedef struct SStmtTableCache { @@ -79,7 +79,7 @@ typedef struct STscStmt { SStmtSQLInfo sql; SStmtExecInfo exec; - SStmtBindInfo bind; + SStmtBindInfo bInfo; } STscStmt; @@ -91,7 +91,6 @@ typedef struct STscStmt { do { \ switch (_newstatus) { \ case STMT_INIT: \ - if ((_stmt)->sql.status != 0) return (_errcode); \ break; \ case STMT_PREPARE: \ if ((_stmt)->sql.status != STMT_INIT) STMT_ERR_RET(_errcode); \ diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 5d1065b59b..9af963a721 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -127,6 +127,10 @@ const char *taos_errstr(TAOS_RES *res) { } void taos_free_result(TAOS_RES *res) { + if (NULL == res) { + return; + } + SRequestObj *pRequest = (SRequestObj *)res; destroyRequest(pRequest); } diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 5a3f232da7..d83c36820d 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -9,12 +9,12 @@ int32_t stmtGetTbName(TAOS_STMT *stmt, char **tbName) { pStmt->sql.type = STMT_TYPE_MULTI_INSERT; - if (NULL == pStmt->bind.tbName) { + if (NULL == pStmt->bInfo.tbName) { tscError("no table name set"); STMT_ERR_RET(TSDB_CODE_TSC_STMT_TBNAME_ERROR); } - *tbName = pStmt->bind.tbName; + *tbName = pStmt->bInfo.tbName; return TSDB_CODE_SUCCESS; } @@ -22,10 +22,10 @@ int32_t stmtGetTbName(TAOS_STMT *stmt, char **tbName) { int32_t stmtSetBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags) { STscStmt* pStmt = (STscStmt*)stmt; - pStmt->bind.tbUid = pTableMeta->uid; - pStmt->bind.tbSuid = pTableMeta->suid; - pStmt->bind.tbType = pTableMeta->tableType; - pStmt->bind.boundTags = tags; + pStmt->bInfo.tbUid = pTableMeta->uid; + pStmt->bInfo.tbSuid = pTableMeta->suid; + pStmt->bInfo.tbType = pTableMeta->tableType; + pStmt->bInfo.boundTags = tags; return TSDB_CODE_SUCCESS; } @@ -54,11 +54,11 @@ int32_t stmtCacheBlock(STscStmt *pStmt) { } uint64_t uid; - if (TSDB_CHILD_TABLE == pStmt->bind.tbType) { - uid = pStmt->bind.tbSuid; + if (TSDB_CHILD_TABLE == pStmt->bInfo.tbType) { + uid = pStmt->bInfo.tbSuid; } else { - ASSERT(TSDB_NORMAL_TABLE == pStmt->bind.tbType); - uid = pStmt->bind.tbUid; + ASSERT(TSDB_NORMAL_TABLE == pStmt->bInfo.tbType); + uid = pStmt->bInfo.tbUid; } if (taosHashGet(pStmt->sql.pTableCache, &uid, sizeof(uid))) { @@ -72,14 +72,14 @@ int32_t stmtCacheBlock(STscStmt *pStmt) { SStmtTableCache cache = { .pDataBlock = pDst, - .boundTags = pStmt->bind.boundTags, + .boundTags = pStmt->bInfo.boundTags, }; if (taosHashPut(pStmt->sql.pTableCache, &uid, sizeof(uid), &cache, sizeof(cache))) { return TSDB_CODE_OUT_OF_MEMORY; } - pStmt->bind.boundTags = NULL; + pStmt->bInfo.boundTags = NULL; return TSDB_CODE_SUCCESS; } @@ -95,7 +95,7 @@ int32_t stmtParseSql(STscStmt* pStmt) { STMT_ERR_RET(parseSql(pStmt->exec.pRequest, false, &pStmt->sql.pQuery, &stmtCb)); - pStmt->bind.needParse = false; + pStmt->bInfo.needParse = false; switch (nodeType(pStmt->sql.pQuery->pRoot)) { case QUERY_NODE_VNODE_MODIF_STMT: @@ -116,14 +116,14 @@ int32_t stmtParseSql(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } int32_t stmtCleanBindInfo(STscStmt* pStmt) { - pStmt->bind.tbUid = 0; - pStmt->bind.tbSuid = 0; - pStmt->bind.tbType = 0; - pStmt->bind.needParse = true; + pStmt->bInfo.tbUid = 0; + pStmt->bInfo.tbSuid = 0; + pStmt->bInfo.tbType = 0; + pStmt->bInfo.needParse = true; - taosMemoryFreeClear(pStmt->bind.tbName); - destroyBoundColumnInfo(pStmt->bind.boundTags); - taosMemoryFreeClear(pStmt->bind.boundTags); + taosMemoryFreeClear(pStmt->bInfo.tbName); + destroyBoundColumnInfo(pStmt->bInfo.boundTags); + taosMemoryFreeClear(pStmt->bInfo.boundTags); return TSDB_CODE_SUCCESS; } @@ -137,7 +137,7 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable) { STableDataBlocks* pBlocks = *(STableDataBlocks**)pIter; uint64_t *key = taosHashGetKey(pIter, NULL); - if (keepTable && (*key == pStmt->bind.tbUid)) { + if (keepTable && (*key == pStmt->bInfo.tbUid)) { qResetStmtDataBlock(pBlocks, true); pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter); @@ -197,10 +197,10 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { STableMeta *pTableMeta = NULL; SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp); - STMT_ERR_RET(catalogGetTableMeta(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->bind.sname, &pTableMeta)); + STMT_ERR_RET(catalogGetTableMeta(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->bInfo.sname, &pTableMeta)); - if (pTableMeta->uid == pStmt->bind.tbUid) { - pStmt->bind.needParse = false; + if (pTableMeta->uid == pStmt->bInfo.tbUid) { + pStmt->bInfo.needParse = false; return TSDB_CODE_SUCCESS; } @@ -212,29 +212,29 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR); } - pStmt->bind.needParse = false; + pStmt->bInfo.needParse = false; - pStmt->bind.tbUid = pTableMeta->uid; - pStmt->bind.tbSuid = pTableMeta->suid; - pStmt->bind.tbType = pTableMeta->tableType; - pStmt->bind.boundTags = pCache->boundTags; + pStmt->bInfo.tbUid = pTableMeta->uid; + pStmt->bInfo.tbSuid = pTableMeta->suid; + pStmt->bInfo.tbType = pTableMeta->tableType; + pStmt->bInfo.boundTags = pCache->boundTags; return TSDB_CODE_SUCCESS; } SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &pTableMeta->uid, sizeof(pTableMeta->uid)); if (pCache) { - pStmt->bind.needParse = false; + pStmt->bInfo.needParse = false; - pStmt->bind.tbUid = pTableMeta->uid; - pStmt->bind.tbSuid = pTableMeta->suid; - pStmt->bind.tbType = pTableMeta->tableType; - pStmt->bind.boundTags = pCache->boundTags; + pStmt->bInfo.tbUid = pTableMeta->uid; + pStmt->bInfo.tbSuid = pTableMeta->suid; + pStmt->bInfo.tbType = pTableMeta->tableType; + pStmt->bInfo.boundTags = pCache->boundTags; STableDataBlocks* pNewBlock = NULL; STMT_ERR_RET(qRebuildStmtDataBlock(&pNewBlock, pCache->pDataBlock)); - if (taosHashPut(pStmt->exec.pBlockHash, &pStmt->bind.tbUid, sizeof(pStmt->bind.tbUid), &pNewBlock, POINTER_BYTES)) { + if (taosHashPut(pStmt->exec.pBlockHash, &pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid), &pNewBlock, POINTER_BYTES)) { STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -261,11 +261,11 @@ TAOS_STMT *stmtInit(TAOS *taos) { taosMemoryFree(pStmt); return NULL; } - - pStmt->taos = pObj; - pStmt->sql.status = STMT_INIT; - pStmt->bind.needParse = true; + pStmt->taos = pObj; + pStmt->bInfo.needParse = true; + pStmt->sql.status = STMT_INIT; + return pStmt; } @@ -290,15 +290,15 @@ int stmtSetTbName(TAOS_STMT *stmt, const char *tbName) { STMT_SWITCH_STATUS(pStmt, STMT_SETTBNAME, TSDB_CODE_TSC_STMT_API_ERROR); - taosMemoryFree(pStmt->bind.tbName); + taosMemoryFree(pStmt->bInfo.tbName); if (NULL == pStmt->exec.pRequest) { STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest)); } - STMT_ERR_RET(qCreateSName(&pStmt->bind.sname, tbName, pStmt->taos->acctId, pStmt->exec.pRequest->pDb, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); + STMT_ERR_RET(qCreateSName(&pStmt->bInfo.sname, tbName, pStmt->taos->acctId, pStmt->exec.pRequest->pDb, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); - pStmt->bind.tbName = strdup(tbName); + pStmt->bInfo.tbName = strdup(tbName); STMT_ERR_RET(stmtGetFromCache(pStmt)); @@ -310,17 +310,17 @@ int stmtSetTbTags(TAOS_STMT *stmt, TAOS_BIND_v2 *tags) { STMT_SWITCH_STATUS(pStmt, STMT_SETTBNAME, TSDB_CODE_TSC_STMT_API_ERROR); - if (pStmt->bind.needParse) { + if (pStmt->bInfo.needParse) { STMT_ERR_RET(stmtParseSql(pStmt)); } - STableDataBlocks *pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bind.tbUid, sizeof(pStmt->bind.tbUid)); + STableDataBlocks *pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid)); if (NULL == pDataBlock) { - tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bind.tbUid); + tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid); STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - STMT_ERR_RET(qBindStmtTagsValue(pDataBlock, pStmt->bind.boundTags, pStmt->bind.tbSuid, &pStmt->bind.sname, tags, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); + STMT_ERR_RET(qBindStmtTagsValue(pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.tbSuid, &pStmt->bInfo.sname, tags, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); return TSDB_CODE_SUCCESS; } @@ -331,7 +331,7 @@ int32_t stmtFetchTagFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fiel STMT_SWITCH_STATUS(pStmt, STMT_FETCH_TAG_FIELDS, TSDB_CODE_TSC_STMT_API_ERROR); - if (pStmt->bind.needParse) { + if (pStmt->bInfo.needParse) { STMT_ERR_RET(stmtParseSql(pStmt)); } @@ -340,13 +340,13 @@ int32_t stmtFetchTagFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fiel STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); } - STableDataBlocks *pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bind.tbUid, sizeof(pStmt->bind.tbUid)); + STableDataBlocks *pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid)); if (NULL == pDataBlock) { - tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bind.tbUid); + tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid); STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - STMT_ERR_RET(qBuildStmtTagFields(pDataBlock, pStmt->bind.boundTags, fieldNum, fields)); + STMT_ERR_RET(qBuildStmtTagFields(pDataBlock, pStmt->bInfo.boundTags, fieldNum, fields)); return TSDB_CODE_SUCCESS; } @@ -356,7 +356,7 @@ int32_t stmtFetchColFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fiel STMT_SWITCH_STATUS(pStmt, STMT_FETCH_COL_FIELDS, TSDB_CODE_TSC_STMT_API_ERROR); - if (pStmt->bind.needParse) { + if (pStmt->bInfo.needParse) { STMT_ERR_RET(stmtParseSql(pStmt)); } @@ -365,9 +365,9 @@ int32_t stmtFetchColFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fiel STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); } - STableDataBlocks *pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bind.tbUid, sizeof(pStmt->bind.tbUid)); + STableDataBlocks *pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid)); if (NULL == pDataBlock) { - tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bind.tbUid); + tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid); STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } @@ -381,21 +381,21 @@ int stmtBindBatch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) { STMT_SWITCH_STATUS(pStmt, STMT_BIND, TSDB_CODE_TSC_STMT_API_ERROR); - if (pStmt->bind.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { - pStmt->bind.needParse = false; + if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { + pStmt->bInfo.needParse = false; } if (NULL == pStmt->exec.pRequest) { STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest)); } - if (pStmt->bind.needParse) { + if (pStmt->bInfo.needParse) { STMT_ERR_RET(stmtParseSql(pStmt)); } - STableDataBlocks *pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bind.tbUid, sizeof(pStmt->bind.tbUid)); + STableDataBlocks *pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid)); if (NULL == pDataBlock) { - tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bind.tbUid); + tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid); STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index de1050850d..17ef2076ec 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include "../../../include/client/taos.h" @@ -15,19 +16,23 @@ typedef struct { void taosMsleep(int mseconds); -unsigned long long getCurrentTime(){ - struct timeval tv; - if (taosGetTimeOfDay(&tv) != 0) { - perror("Failed to get current time in ms"); - exit(EXIT_FAILURE); - } +int32_t taosGetTimeOfDay(struct timeval *tv) { + return gettimeofday(tv, NULL); +} +void *taosMemoryCalloc(int32_t num, int32_t size) { + return calloc(num, size); +} +void taosMemoryFree(const void *ptr) { + if (ptr == NULL) return; - return (uint64_t)tv.tv_sec * 1000000ULL + (uint64_t)tv.tv_usec; + return free((void*)ptr); } - - - +static int64_t taosGetTimestampUs() { + struct timeval systemTime; + taosGetTimeOfDay(&systemTime); + return (int64_t)systemTime.tv_sec * 1000000L + (int64_t)systemTime.tv_usec; +} int stmt_scol_func1(TAOS_STMT *stmt) { struct { @@ -42,43 +47,50 @@ int stmt_scol_func1(TAOS_STMT *stmt) { char bin[40]; char blob[80]; } v = {0}; + int32_t len[10] = {sizeof(v.ts), sizeof(v.v1), sizeof(v.v2), sizeof(v.f4), sizeof(v.bin), sizeof(v.bin)}; - TAOS_BIND params[10]; + TAOS_BIND_v2 params[10]; params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; params[0].buffer_length = sizeof(v.ts); params[0].buffer = &v.ts; - params[0].length = ¶ms[0].buffer_length; + params[0].length = &len[0]; params[0].is_null = NULL; + params[0].num = 1; params[1].buffer_type = TSDB_DATA_TYPE_TINYINT; params[1].buffer_length = sizeof(v.v1); params[1].buffer = &v.v1; - params[1].length = ¶ms[1].buffer_length; + params[1].length = &len[1]; params[1].is_null = NULL; + params[1].num = 1; params[2].buffer_type = TSDB_DATA_TYPE_SMALLINT; params[2].buffer_length = sizeof(v.v2); params[2].buffer = &v.v2; - params[2].length = ¶ms[2].buffer_length; + params[2].length = &len[2]; params[2].is_null = NULL; + params[2].num = 1; params[3].buffer_type = TSDB_DATA_TYPE_FLOAT; params[3].buffer_length = sizeof(v.f4); params[3].buffer = &v.f4; - params[3].length = ¶ms[3].buffer_length; + params[3].length = &len[3]; params[3].is_null = NULL; + params[3].num = 1; params[4].buffer_type = TSDB_DATA_TYPE_BINARY; params[4].buffer_length = sizeof(v.bin); params[4].buffer = v.bin; - params[4].length = ¶ms[4].buffer_length; + params[4].length = &len[4]; params[4].is_null = NULL; + params[4].num = 1; params[5].buffer_type = TSDB_DATA_TYPE_BINARY; params[5].buffer_length = sizeof(v.bin); params[5].buffer = v.bin; - params[5].length = ¶ms[5].buffer_length; + params[5].length = &len[5]; params[5].is_null = NULL; + params[5].num = 1; char *sql = "insert into ? (ts, v1,v2,f4,bin,bin2) values(?,?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); @@ -122,7 +134,7 @@ int stmt_scol_func1(TAOS_STMT *stmt) { return 0; } - +#if 0 int stmt_scol_func2(TAOS_STMT *stmt) { struct { @@ -4192,7 +4204,7 @@ int stmt_funcb_sc3(TAOS_STMT *stmt) { return 0; } - +#endif void check_result(TAOS *taos, char *tname, int printr, int expected) { char sql[255] = "SELECT * FROM "; @@ -4263,7 +4275,7 @@ int sql_perf1(TAOS *taos) { } - unsigned long long starttime = getCurrentTime(); + int64_t starttime = taosGetTimestampUs(); for (int i = 0; i < 3000; ++i) { result = taos_query(taos, sql[i]); int code = taos_errno(result); @@ -4275,7 +4287,7 @@ int sql_perf1(TAOS *taos) { taos_free_result(result); } - unsigned long long endtime = getCurrentTime(); + int64_t endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%.1f useconds\n", 3000*120*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*120*60)); for (int i = 0; i < 3000; i++) { @@ -4291,7 +4303,7 @@ int sql_perf1(TAOS *taos) { //one table 60 records one time int sql_perf_s1(TAOS *taos) { - char **sql = taosMemoryCalloc(1, sizeof(char*) * 360000); + char **sql = calloc(1, sizeof(char*) * 360000); TAOS_RES *result; for (int i = 0; i < 360000; i++) { @@ -4316,7 +4328,7 @@ int sql_perf_s1(TAOS *taos) { } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); for (int i = 0; i < 360000; ++i) { result = taos_query(taos, sql[i]); int code = taos_errno(result); @@ -4328,7 +4340,7 @@ int sql_perf_s1(TAOS *taos) { taos_free_result(result); } - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%.1f useconds\n", 3000*120*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*120*60)); for (int i = 0; i < 360000; i++) { @@ -4363,7 +4375,7 @@ int sql_s_perf1(TAOS *taos) { } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); for (int i = 0; i < 3000; ++i) { result = taos_query(taos, sql[i]); int code = taos_errno(result); @@ -4375,7 +4387,7 @@ int sql_s_perf1(TAOS *taos) { taos_free_result(result); } - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%.1f useconds\n", 3000*120*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*120*60)); for (int i = 0; i < 3000; i++) { @@ -4528,7 +4540,7 @@ void* runcase(void *par) { taos_stmt_close(stmt); #endif - +#if 0 #if 1 prepare(taos, 1, 1); @@ -5024,6 +5036,7 @@ void* runcase(void *par) { printf("check result end\n"); taos_stmt_close(stmt); +#endif #endif printf("test end\n"); @@ -5066,11 +5079,11 @@ int main(int argc, char *argv[]) exit(1); } - TdThread *pThreadList = (TdThread *) taosMemoryCalloc(sizeof(TdThread), 4); + pthread_t *pThreadList = (pthread_t *) taosMemoryCalloc(sizeof(pthread_t), 4); - TdThreadAttr thattr; - taosThreadAttrInit(&thattr); - taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); + pthread_attr_t thattr; + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); T_par par[4]; par[0].taos = taos[0]; @@ -5082,13 +5095,13 @@ int main(int argc, char *argv[]) par[3].taos = taos[3]; par[3].idx = 3; - taosThreadCreate(&(pThreadList[0]), &thattr, runcase, (void *)&par[0]); - //taosThreadCreate(&(pThreadList[1]), &thattr, runcase, (void *)&par[1]); - //taosThreadCreate(&(pThreadList[2]), &thattr, runcase, (void *)&par[2]); - //taosThreadCreate(&(pThreadList[3]), &thattr, runcase, (void *)&par[3]); + pthread_create(&(pThreadList[0]), &thattr, runcase, (void *)&par[0]); + //pthread_create(&(pThreadList[1]), &thattr, runcase, (void *)&par[1]); + //pthread_create(&(pThreadList[2]), &thattr, runcase, (void *)&par[2]); + //pthread_create(&(pThreadList[3]), &thattr, runcase, (void *)&par[3]); while(1) { - taosSsleep(1); + sleep(1); } return 0; } From e2c6bf41df1d33ba3698fe0aef0f32b3e52bec0a Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 18 Apr 2022 13:36:01 +0800 Subject: [PATCH 11/47] refactor: add compare logic for json value --- include/common/tdatablock.h | 20 +- source/libs/scalar/src/sclvector.c | 289 +++++++++++++++++------------ 2 files changed, 186 insertions(+), 123 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 15f3246013..8b9cf74d7a 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -54,12 +54,15 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet); BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \ } while (0) +#define colDataIsNull_var(pColumnInfoData, row) (pColumnInfoData->varmeta.offset[row] == -1) +#define colDataSetNull_var(pColumnInfoData, row) (pColumnInfoData->varmeta.offset[row] = -1) + static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) { if (!pColumnInfoData->hasNull) { return false; } if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { - return pColumnInfoData->varmeta.offset[row] == -1; + return colDataIsNull_var(pColumnInfoData, row); } else { if (pColumnInfoData->nullbitmap == NULL) { return false; @@ -86,7 +89,7 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u } if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { - return pColumnInfoData->varmeta.offset[row] == -1; + return colDataIsNull_var(pColumnInfoData, row); } else { if (pColumnInfoData->nullbitmap == NULL) { return false; @@ -98,15 +101,18 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u #define BitmapLen(_n) (((_n) + ((1 << NBIT) - 1)) >> NBIT) +#define colDataGetVarData(p1_, r_) ((p1_)->pData + (p1_)->varmeta.offset[(r_)]) + +#define colDataGetNumData(p1_, r_) ((p1_)->pData + ((r_) * (p1_)->info.bytes)) // SColumnInfoData, rowNumber -#define colDataGetData(p1_, r_) \ - ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? ((p1_)->pData + (p1_)->varmeta.offset[(r_)]) \ - : ((p1_)->pData + ((r_) * (p1_)->info.bytes))) +#define colDataGetData(p1_, r_) \ + ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? colDataGetVarData(p1_, r_) \ + : colDataGetNumData(p1_, r_) static FORCE_INLINE void colDataAppendNULL(SColumnInfoData* pColumnInfoData, uint32_t currentRow) { // There is a placehold for each NULL value of binary or nchar type. if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { - pColumnInfoData->varmeta.offset[currentRow] = -1; // it is a null value of VAR type. + colDataSetNull_var(pColumnInfoData, currentRow); // it is a null value of VAR type. } else { colDataSetNull_f(pColumnInfoData->nullbitmap, currentRow); } @@ -117,7 +123,7 @@ static FORCE_INLINE void colDataAppendNULL(SColumnInfoData* pColumnInfoData, uin static FORCE_INLINE void colDataAppendNNULL(SColumnInfoData* pColumnInfoData, uint32_t start, size_t nRows) { if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { for (int32_t i = start; i < start + nRows; ++i) { - pColumnInfoData->varmeta.offset[i] = -1; // it is a null value of VAR type. + colDataSetNull_var(pColumnInfoData,i); // it is a null value of VAR type. } } else { for (int32_t i = start; i < start + nRows; ++i) { diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 11dc60dab3..aa51c9b631 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -25,6 +25,9 @@ #include "tdatablock.h" #include "ttypes.h" +#define LEFT_COL ((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData)) +#define RIGHT_COL ((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData)) + typedef int64_t (*_getBigintValue_fn_t)(void *src, int32_t index); int64_t getVectorBigintValue_TINYINT(void *src, int32_t index) { @@ -189,65 +192,6 @@ static FORCE_INLINE void varToBool(char *buf, SScalarParam* pOut, int32_t rowInd // taosMemoryFree(t); //} -//TODO opt performance, tmp is not needed. -int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, int32_t inType, int32_t outType) { - int32_t bufSize = pIn->columnData->info.bytes; - char *tmp = taosMemoryMalloc(bufSize + VARSTR_HEADER_SIZE); - -// bool vton = false; - - _bufConverteFunc func = NULL; - if (TSDB_DATA_TYPE_BOOL == outType) { - func = varToBool; - } else if (IS_SIGNED_NUMERIC_TYPE(outType) || TSDB_DATA_TYPE_TIMESTAMP == outType) { - func = varToSigned; - } else if (IS_UNSIGNED_NUMERIC_TYPE(outType)) { - func = varToUnsigned; - } else if (IS_FLOAT_TYPE(outType)) { - func = varToFloat; -// } else if (outType == TSDB_DATA_TYPE_NCHAR) { // can not be nchar or binary -// func = varToNchar; -// vton = true; - } else { - sclError("invalid convert outType:%d", outType); - return TSDB_CODE_QRY_APP_ERROR; - } - - pOut->numOfRows = pIn->numOfRows; - for (int32_t i = 0; i < pIn->numOfRows; ++i) { - if (colDataIsNull_s(pIn->columnData, i)) { - colDataAppendNULL(pOut->columnData, i); - continue; - } - - char* data = colDataGetData(pIn->columnData, i); -// if (vton) { -// memcpy(tmp, data, varDataTLen(data)); -// } else { - if (TSDB_DATA_TYPE_VARCHAR == inType) { - memcpy(tmp, varDataVal(data), varDataLen(data)); - tmp[varDataLen(data)] = 0; - } else { - ASSERT(varDataLen(data) <= bufSize); - - int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(data), varDataLen(data), tmp); - if (len < 0) { - sclError("castConvert taosUcs4ToMbs error 1"); - taosMemoryFreeClear(tmp); - return TSDB_CODE_QRY_APP_ERROR; - } - - tmp[len] = 0; - } -// } - - (*func)(tmp, pOut, i); - } - - taosMemoryFreeClear(tmp); - return TSDB_CODE_SUCCESS; -} - void convertNumberToNumber(const void *inData, void *outData, int8_t inType, int8_t outType){ switch (outType) { case TSDB_DATA_TYPE_BOOL: { @@ -303,18 +247,13 @@ void convertNumberToNumber(const void *inData, void *outData, int8_t inType, int void convertStringToDouble(const void *inData, void *outData, int8_t inType, int8_t outType){ char *tmp = taosMemoryMalloc(varDataTLen(inData)); - if (inType == TSDB_DATA_TYPE_NCHAR) { - int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(inData), varDataLen(inData), tmp); - if (len < 0) { - sclError("castConvert taosUcs4ToMbs error 1"); - } - - tmp[len] = 0; - } else { - memcpy(tmp, varDataVal(inData), varDataLen(inData)); - tmp[varDataLen(inData)] = 0; + int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(inData), varDataLen(inData), tmp); + if (len < 0) { + sclError("castConvert taosUcs4ToMbs error 1"); } + tmp[len] = 0; + ASSERT(outType == TSDB_DATA_TYPE_DOUBLE); double value = strtod(tmp, NULL); @@ -322,11 +261,85 @@ void convertStringToDouble(const void *inData, void *outData, int8_t inType, int taosMemoryFreeClear(tmp); } +//TODO opt performance, tmp is not needed. +int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, int32_t inType, int32_t outType) { + ASSERT(!IS_VAR_DATA_TYPE(outType)); + int32_t bufSize = pIn->columnData->info.bytes; + char *tmp = taosMemoryMalloc(bufSize + VARSTR_HEADER_SIZE); + +// bool vton = false; + + _bufConverteFunc func = NULL; + if (TSDB_DATA_TYPE_BOOL == outType) { + func = varToBool; + } else if (IS_SIGNED_NUMERIC_TYPE(outType) || TSDB_DATA_TYPE_TIMESTAMP == outType) { + func = varToSigned; + } else if (IS_UNSIGNED_NUMERIC_TYPE(outType)) { + func = varToUnsigned; + } else if (IS_FLOAT_TYPE(outType)) { + func = varToFloat; +// } else if (outType == TSDB_DATA_TYPE_NCHAR) { // can not be nchar or binary +// func = varToNchar; +// vton = true; + } else { + sclError("invalid convert outType:%d", outType); + return TSDB_CODE_QRY_APP_ERROR; + } + + pOut->numOfRows = pIn->numOfRows; + for (int32_t i = 0; i < pIn->numOfRows; ++i) { + if (colDataIsNull_s(pIn->columnData, i)) { + colDataAppendNULL(pOut->columnData, i); + continue; + } + + char* data = colDataGetVarData(pIn->columnData, i); + int32_t convertType = inType; + if(inType == TSDB_DATA_TYPE_JSON){ + if(*data == TSDB_DATA_TYPE_NULL) { + colDataAppendNULL(pOut->columnData, i); + continue; + } + else if(*data == TSDB_DATA_TYPE_NCHAR) { + data += CHAR_BYTES; + convertType = TSDB_DATA_TYPE_NCHAR; + } else { + convertNumberToNumber(data+CHAR_BYTES, colDataGetNumData(pOut->columnData, i), *data, outType); + continue; + } + } +// if (vton) { +// memcpy(tmp, data, varDataTLen(data)); +// } else { + if (TSDB_DATA_TYPE_VARCHAR == convertType) { + memcpy(tmp, varDataVal(data), varDataLen(data)); + tmp[varDataLen(data)] = 0; + } else if (TSDB_DATA_TYPE_NCHAR == convertType){ + ASSERT(varDataLen(data) <= bufSize); + + int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(data), varDataLen(data), tmp); + if (len < 0) { + sclError("castConvert taosUcs4ToMbs error 1"); + taosMemoryFreeClear(tmp); + return TSDB_CODE_QRY_APP_ERROR; + } + + tmp[len] = 0; + } +// } + + (*func)(tmp, pOut, i); + } + + taosMemoryFreeClear(tmp); + return TSDB_CODE_SUCCESS; +} + double getVectorDoubleValue_JSON(void *src, int32_t index){ - ASSERT(!colDataIsNull_s(((SColumnInfoData*)src), index)); - char *data = colDataGetData((SColumnInfoData*)src, index); + ASSERT(!colDataIsNull_var(((SColumnInfoData*)src), index)); + char *data = colDataGetVarData((SColumnInfoData*)src, index); double out = 0; - if(*data == TSDB_DATA_TYPE_NCHAR) { + if(*data == TSDB_DATA_TYPE_NCHAR) { // json inner type can not be BINARY convertStringToDouble(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE); } { convertNumberToNumber(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE); @@ -685,8 +698,12 @@ static void vectorMathAddHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRig colDataAppendNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { - *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i) - + getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), 0); + if (colDataIsNull_s(pLeftCol, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore + } + *output = getVectorDoubleValueFnLeft(LEFT_COL, i) + + getVectorDoubleValueFnRight(RIGHT_COL, 0); } pOutputCol->hasNull = pLeftCol->hasNull; if (pOutputCol->hasNull) { @@ -729,17 +746,17 @@ void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO pOut->numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows); - char *pRightData = colDataGetData(pRight->columnData, 0); + char *pRightData = colDataGetVarData(pRight->columnData, 0); for (; i >= 0 && i < pLeft->numOfRows; i += step) { - if (pLeft->columnData->varmeta.offset[i] == -1) { - pOutputCol->varmeta.offset[i] = -1; + if (colDataIsNull_var(pLeft->columnData, i)) { + colDataSetNull_var(pOutputCol, i); pOutputCol->hasNull = true; continue; } - char *pLeftData = colDataGetData(pLeft->columnData, i); + char *pLeftData = colDataGetVarData(pLeft->columnData, i); char *value = getJsonValue(pLeftData, pRightData); if (!value || *value == TSDB_DATA_TYPE_NULL) { - pOutputCol->varmeta.offset[i] = -1; + colDataSetNull_var(pOutputCol, i); pOutputCol->hasNull = true; continue; } @@ -769,8 +786,8 @@ void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } - *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i) - + getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i); + *output = getVectorDoubleValueFnLeft(LEFT_COL, i) + + getVectorDoubleValueFnRight(RIGHT_COL, i); } pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull); @@ -802,8 +819,12 @@ static void vectorMathSubHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRig colDataAppendNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { - *output = (getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i) - - getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), 0)) * factor; + if (colDataIsNull_s(pLeftCol, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore + } + *output = (getVectorDoubleValueFnLeft(LEFT_COL, i) + - getVectorDoubleValueFnRight(RIGHT_COL, 0)) * factor; } pOutputCol->hasNull = pLeftCol->hasNull; if (pOutputCol->hasNull) { @@ -834,8 +855,8 @@ void vectorMathSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } - *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i) - - getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i); + *output = getVectorDoubleValueFnLeft(LEFT_COL, i) + - getVectorDoubleValueFnRight(RIGHT_COL, i); } pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull); @@ -867,8 +888,12 @@ static void vectorMathMultiplyHelper(SColumnInfoData* pLeftCol, SColumnInfoData* colDataAppendNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { - *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i) - * getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), 0); + if (colDataIsNull_s(pLeftCol, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore + } + *output = getVectorDoubleValueFnLeft(LEFT_COL, i) + * getVectorDoubleValueFnRight(RIGHT_COL, 0); } pOutputCol->hasNull = pLeftCol->hasNull; if (pOutputCol->hasNull) { @@ -898,8 +923,8 @@ void vectorMathMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } - *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i) - * getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i); + *output = getVectorDoubleValueFnLeft(LEFT_COL, i) + * getVectorDoubleValueFnRight(RIGHT_COL, i); } pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull); @@ -941,8 +966,8 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } - *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i) - /getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i); + *output = getVectorDoubleValueFnLeft(LEFT_COL, i) + /getVectorDoubleValueFnRight(RIGHT_COL, i); } pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull); @@ -954,12 +979,16 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p } } else if (pLeft->numOfRows == 1) { - if (colDataIsNull_f(pLeftCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value + if (colDataIsNull_s(pLeftCol, 0)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows); } else { for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) { - *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), 0) - / getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i); + if (colDataIsNull_s(pRightCol, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore + } + *output = getVectorDoubleValueFnLeft(LEFT_COL, 0) + / getVectorDoubleValueFnRight(RIGHT_COL, i); } pOutputCol->hasNull = pRightCol->hasNull; if (pOutputCol->hasNull) { @@ -967,12 +996,16 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p } } } else if (pRight->numOfRows == 1) { - if (colDataIsNull_f(pRightCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value + if (colDataIsNull_s(pRightCol, 0)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows); } else { for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) { - *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i) - / getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), 0); + if (colDataIsNull_s(pLeftCol, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore + } + *output = getVectorDoubleValueFnLeft(LEFT_COL, i) + / getVectorDoubleValueFnRight(RIGHT_COL, 0); } pOutputCol->hasNull = pLeftCol->hasNull; if (pOutputCol->hasNull) { @@ -1004,13 +1037,13 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - if (colDataIsNull_f(pLeftCol->nullbitmap, i) || colDataIsNull_f(pRightCol->nullbitmap, i)) { + if (colDataIsNull_s(pLeftCol, i) || colDataIsNull_s(pRightCol, i)) { colDataAppendNULL(pOutputCol, i); continue; } - double lx = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i); - double rx = getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i); + double lx = getVectorDoubleValueFnLeft(LEFT_COL, i); + double rx = getVectorDoubleValueFnRight(RIGHT_COL, i); if (isnan(lx) || isinf(lx) || isnan(rx) || isinf(rx)) { colDataAppendNULL(pOutputCol, i); continue; @@ -1019,17 +1052,17 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *output = lx - ((int64_t)(lx / rx)) * rx; } } else if (pLeft->numOfRows == 1) { - double lx = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), 0); - if (colDataIsNull_f(pLeftCol->nullbitmap, 0) || isnan(lx) || isinf(lx)) { // Set pLeft->numOfRows NULL value + double lx = getVectorDoubleValueFnLeft(LEFT_COL, 0); + if (colDataIsNull_s(pLeftCol, 0) || isnan(lx) || isinf(lx)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows); } else { for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) { - if (colDataIsNull_f(pRightCol->nullbitmap, i)) { + if (colDataIsNull_s(pRightCol, i)) { colDataAppendNULL(pOutputCol, i); continue; } - double rx = getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i); + double rx = getVectorDoubleValueFnRight(RIGHT_COL, i); if (isnan(rx) || isinf(rx) || FLT_EQUAL(rx, 0)) { colDataAppendNULL(pOutputCol, i); continue; @@ -1039,17 +1072,17 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam } } } else if (pRight->numOfRows == 1) { - double rx = getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), 0); - if (colDataIsNull_f(pRightCol->nullbitmap, 0) || FLT_EQUAL(rx, 0)) { // Set pLeft->numOfRows NULL value + double rx = getVectorDoubleValueFnRight(RIGHT_COL, 0); + if (colDataIsNull_s(pRightCol, 0) || FLT_EQUAL(rx, 0)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows); } else { for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) { - if (colDataIsNull_f(pLeftCol->nullbitmap, i)) { + if (colDataIsNull_s(pLeftCol, i)) { colDataAppendNULL(pOutputCol, i); continue; } - double lx = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i); + double lx = getVectorDoubleValueFnLeft(LEFT_COL, i); if (isnan(lx) || isinf(lx)) { colDataAppendNULL(pOutputCol, i); continue; @@ -1083,7 +1116,7 @@ void vectorMathMinus(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } - *output = - getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i); + *output = - getVectorDoubleValueFnLeft(LEFT_COL, i); } pOutputCol->hasNull = pLeftCol->hasNull; @@ -1151,11 +1184,15 @@ static void vectorBitAndHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRigh double *output = (double *)pOutputCol->pData; - if (colDataIsNull_f(pRightCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value + if (colDataIsNull_s(pRightCol, 0)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { - *output = getVectorBigintValueFnLeft(pLeftCol->pData, i) & getVectorBigintValueFnRight(pRightCol->pData, 0); + if (colDataIsNull_s(pLeftCol, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore + } + *output = getVectorBigintValueFnLeft(LEFT_COL, i) & getVectorBigintValueFnRight(RIGHT_COL, 0); } pOutputCol->hasNull = pLeftCol->hasNull; if (pOutputCol->hasNull) { @@ -1181,7 +1218,11 @@ void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int64_t *output = (int64_t *)pOutputCol->pData; if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - *output = getVectorBigintValueFnLeft(pLeftCol->pData, i) & getVectorBigintValueFnRight(pRightCol->pData, i); + if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore + } + *output = getVectorBigintValueFnLeft(LEFT_COL, i) & getVectorBigintValueFnRight(RIGHT_COL, i); } pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull); @@ -1208,12 +1249,16 @@ static void vectorBitOrHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRight int64_t *output = (int64_t *)pOutputCol->pData; - if (colDataIsNull_f(pRightCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value + if (colDataIsNull_s(pRightCol, 0)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, numOfRows); } else { - int64_t rx = getVectorBigintValueFnRight(pRightCol->pData, 0); + int64_t rx = getVectorBigintValueFnRight(RIGHT_COL, 0); for (; i >= 0 && i < numOfRows; i += step, output += 1) { - *output = getVectorBigintValueFnLeft(pLeftCol->pData, i) | rx; + if (colDataIsNull_s(pLeftCol, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore + } + *output = getVectorBigintValueFnLeft(LEFT_COL, i) | rx; } pOutputCol->hasNull = pLeftCol->hasNull; if (pOutputCol->hasNull) { @@ -1239,7 +1284,11 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int64_t *output = (int64_t *)pOutputCol->pData; if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - *output = getVectorBigintValueFnLeft(pLeftCol->pData, i) | getVectorBigintValueFnRight(pRightCol->pData, i); + if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore + } + *output = getVectorBigintValueFnLeft(LEFT_COL, i) | getVectorBigintValueFnRight(RIGHT_COL, i); } pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull); @@ -1407,6 +1456,10 @@ void vectorJsonContains(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { for(int32_t i = 0; i < pLeft->numOfRows; ++i) { int8_t v = colDataIsNull_s(pLeft->columnData, i)? 1:0; + if (v && pLeft->columnData->info.type == TSDB_DATA_TYPE_JSON){ + char *data = colDataGetVarData(pLeft->columnData, i); + v = (*data == TSDB_DATA_TYPE_NULL)? 1:0; + } colDataAppendInt8(pOut->columnData, i, &v); } pOut->numOfRows = pLeft->numOfRows; @@ -1415,6 +1468,10 @@ void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, void vectorNotNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { for(int32_t i = 0; i < pLeft->numOfRows; ++i) { int8_t v = colDataIsNull_s(pLeft->columnData, i)? 0:1; + if (v && pLeft->columnData->info.type == TSDB_DATA_TYPE_JSON){ + char *data = colDataGetVarData(pLeft->columnData, i); + v = (*data == TSDB_DATA_TYPE_NULL)? 0:1; + } colDataAppendInt8(pOut->columnData, i, &v); } pOut->numOfRows = pLeft->numOfRows; From d2d29d992ff8c155e09e389191c51ba2713a9209 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 18 Apr 2022 14:25:19 +0800 Subject: [PATCH 12/47] refactor: add compare logic for json value --- include/common/tdatablock.h | 2 +- source/libs/scalar/src/sclvector.c | 26 +++++++++++++++++++++----- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 8b9cf74d7a..e40efd815d 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -107,7 +107,7 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u // SColumnInfoData, rowNumber #define colDataGetData(p1_, r_) \ ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? colDataGetVarData(p1_, r_) \ - : colDataGetNumData(p1_, r_) + : colDataGetNumData(p1_, r_)) static FORCE_INLINE void colDataAppendNULL(SColumnInfoData* pColumnInfoData, uint32_t currentRow) { // There is a placehold for each NULL value of binary or nchar type. diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index aa51c9b631..56787b229b 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -347,7 +347,7 @@ double getVectorDoubleValue_JSON(void *src, int32_t index){ return out; } -void convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t typeRight, char **pLeftData, char **pRightData, void *pLeftOut, void *pRightOut){ +void convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t typeRight, char **pLeftData, char **pRightData, void *pLeftOut, void *pRightOut, bool *isNull){ if(optr == OP_TYPE_JSON_CONTAINS) { return; } @@ -364,6 +364,10 @@ void convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t typeRight = **pRightData; (*pRightData) ++; } + if(typeLeft == TSDB_DATA_TYPE_NULL || typeRight == TSDB_DATA_TYPE_NULL){ + *isNull = true; + return; + } int8_t type = vectorGetConvertType(typeLeft, typeRight); if(type == 0) { @@ -381,7 +385,7 @@ void convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t *pLeftData = pLeftOut; } - if(IS_VAR_DATA_TYPE(typeRight)) { + if(typeRight == TSDB_DATA_TYPE_NCHAR) { convertStringToDouble(*pRightData, pRightOut, typeRight, type); *pRightData = pRightOut; } else if(typeRight != type) { @@ -1339,7 +1343,11 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * int64_t leftOut = 0; int64_t rightOut = 0; - convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut); + bool isJsonnull = false; + convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull); + if(isJsonnull){ + continue; // TODO set null or ignore + } bool res = filterDoCompare(fp, optr, pLeftData, pRightData); colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); } @@ -1354,7 +1362,11 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * char *pRightData = colDataGetData(pRight->columnData, 0); int64_t leftOut = 0; int64_t rightOut = 0; - convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut); + bool isJsonnull = false; + convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull); + if(isJsonnull){ + continue; // TODO set null or ignore + } bool res = filterDoCompare(fp, optr, pLeftData, pRightData); colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); } @@ -1368,7 +1380,11 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * char *pRightData = colDataGetData(pLeft->columnData, i); int64_t leftOut = 0; int64_t rightOut = 0; - convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut); + bool isJsonnull = false; + convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull); + if(isJsonnull){ + continue; // TODO set null or ignore + } bool res = filterDoCompare(fp, optr, pLeftData, pRightData); colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); } From 24f805c5c2ebfbb1ed8fe5dc8c967b210eca84d2 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 18 Apr 2022 15:54:40 +0800 Subject: [PATCH 13/47] refactor: add to_json function --- include/libs/scalar/scalar.h | 1 + include/util/tjson.h | 2 +- source/libs/function/src/builtins.c | 2 +- source/libs/scalar/src/sclfunc.c | 54 +++++++++++++++++++++++++++++ source/util/src/tjson.c | 35 ++++++++++++++++++- 5 files changed, 91 insertions(+), 3 deletions(-) diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index 49af10cd2c..6f4a87b02b 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -76,6 +76,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp /* Time related functions */ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t nowFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); diff --git a/include/util/tjson.h b/include/util/tjson.h index 9f347e5896..28bcb3cfc6 100644 --- a/include/util/tjson.h +++ b/include/util/tjson.h @@ -76,7 +76,7 @@ char* tjsonToString(const SJson* pJson); char* tjsonToUnformattedString(const SJson* pJson); SJson* tjsonParse(const char* pStr); -bool tjsonIsObject(const SJson* pJson); +bool tjsonValidateJson(const char* pJson); #ifdef __cplusplus } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index bba5145812..342eb5990b 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -893,7 +893,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .translateFunc = translateToJson, .getEnvFunc = NULL, .initFunc = NULL, - .sprocessFunc = NULL, + .sprocessFunc = toJsonFunction, .finalizeFunc = NULL } }; diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 97cc80b946..05269989cf 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -4,6 +4,7 @@ #include "ttime.h" #include "sclInt.h" #include "sclvector.h" +#include "tjson.h" typedef float (*_float_fn)(float); typedef double (*_double_fn)(double); @@ -920,6 +921,59 @@ int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarP return TSDB_CODE_SUCCESS; } +int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + int32_t type = GET_PARAM_TYPE(pInput); + if (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR) { + return TSDB_CODE_FAILED; + } + + if (inputNum != 1) { + return TSDB_CODE_FAILED; + } + + char *input = pInput[0].columnData->pData + pInput[0].columnData->varmeta.offset[0]; + char *tmp = taosMemoryCalloc(pInput[0].columnData->info.bytes + 1, 1); + for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { + if (colDataIsNull_s(pInput[0].columnData, i)) { + colDataAppendNULL(pOutput->columnData, i); + continue; + } + + if(type == TSDB_DATA_TYPE_NCHAR){ + if (varDataTLen(input) > TSDB_MAX_JSON_TAG_LEN){ + colDataAppendNULL(pOutput->columnData, i); + continue; + } + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), tmp); + if (len < 0) { + colDataAppendNULL(pOutput->columnData, i); + continue; + } + tmp[len] = 0; + }else{ + if (varDataLen(input) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){ + colDataAppendNULL(pOutput->columnData, i); + continue; + } + memcpy(tmp, varDataVal(input), varDataLen(input)); + tmp[varDataTLen(input)] = 0; + } + + if(!tjsonValidateJson(tmp)){ + colDataAppendNULL(pOutput->columnData, i); + continue; + } + + colDataAppend(pOutput->columnData, i, input, false); + input += varDataTLen(input); + } + taosMemoryFree(tmp); + + pOutput->numOfRows = pInput->numOfRows; + + return TSDB_CODE_SUCCESS; +} + int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int32_t type = GET_PARAM_TYPE(&pInput[0]); int32_t timePrec = GET_PARAM_PRECISON(&pInput[0]); diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index 8df404f85f..59dab76989 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -276,4 +276,37 @@ int32_t tjsonToArray(const SJson* pJson, const char* pName, FToObject func, void } SJson* tjsonParse(const char* pStr) { return cJSON_Parse(pStr); } -bool tjsonIsObject(const SJson* pJson) { return cJSON_IsObject(pJson); } \ No newline at end of file + +bool tjsonValidateJson(const char *jIn) { + if (!jIn){ + return false; + } + + // set json real data + cJSON *root = cJSON_Parse(jIn); + if (root == NULL){ + return false; + } + + if(!cJSON_IsObject(root)){ + return false; + } + int size = cJSON_GetArraySize(root); + for(int i = 0; i < size; i++) { + cJSON* item = cJSON_GetArrayItem(root, i); + if (!item) { + return false; + } + + char* jsonKey = item->string; + if (!jsonKey) return false; + for (size_t j = 0; j < strlen(jsonKey); ++i) { + if (isprint(jsonKey[i]) == 0) return false; + } + + if (item->type == cJSON_Object || item->type == cJSON_Array) { + return false; + } + } + return true; +} \ No newline at end of file From 0c2f22fb82696f775b297569d05e4f54d936822e Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 18 Apr 2022 19:08:27 +0800 Subject: [PATCH 14/47] stmt --- include/libs/parser/parser.h | 2 +- source/client/inc/clientStmt.h | 20 +---- source/client/src/clientMain.c | 8 +- source/client/src/clientStmt.c | 74 ++++++++++++------ source/libs/parser/src/parInsert.c | 26 ++++--- source/libs/parser/src/parInsertData.c | 24 ++++-- tests/script/api/batchprepare.c | 102 ++++++++++++++++++++++--- tests/script/api/makefile | 6 +- 8 files changed, 180 insertions(+), 82 deletions(-) diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 2bdcd8c06c..a7377655c7 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -82,7 +82,7 @@ void qDestroyQuery(SQuery* pQueryNode); int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema); int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash); -void qResetStmtDataBlock(void* pBlock, bool freeData); +int32_t qResetStmtDataBlock(void* block, bool keepBuf); int32_t qCloneStmtDataBlock(void** pDst, void* pSrc); void qFreeStmtDataBlock(void* pDataBlock); int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc); diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index 423d474d2e..07b3e388af 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -33,6 +33,7 @@ typedef enum { STMT_INIT = 1, STMT_PREPARE, STMT_SETTBNAME, + STMT_SETTAGS, STMT_FETCH_TAG_FIELDS, STMT_FETCH_COL_FIELDS, STMT_BIND, @@ -87,25 +88,6 @@ typedef struct STscStmt { #define STMT_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) #define STMT_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) -#define STMT_SWITCH_STATUS(_stmt, _newstatus, _errcode) \ - do { \ - switch (_newstatus) { \ - case STMT_INIT: \ - break; \ - case STMT_PREPARE: \ - if ((_stmt)->sql.status != STMT_INIT) STMT_ERR_RET(_errcode); \ - break; \ - case STMT_SETTBNAME: \ - break; \ - default: \ - STMT_ERR_RET(_errcode); \ - break; \ - } \ - \ - (_stmt)->sql.status = _newstatus; \ - } while (0) - - TAOS_STMT *stmtInit(TAOS *taos); int stmtClose(TAOS_STMT *stmt); int stmtExec(TAOS_STMT *stmt); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index a210b16e7b..8ff5674860 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -655,7 +655,7 @@ int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) { } int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int colIdx) { - return stmtBindBatch(stmt, bind); /* TODO */ + return stmtBindBatch(stmt, bind, colIdx); /* TODO */ } int taos_stmt_add_batch(TAOS_STMT *stmt) { @@ -709,12 +709,6 @@ TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt) { } char *taos_stmt_errstr(TAOS_STMT *stmt) { - if (stmt == NULL) { - tscError("NULL parameter for %s", __FUNCTION__); - terrno = TSDB_CODE_INVALID_PARA; - return NULL; - } - return (char *)stmtErrstr(stmt); } diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index d83c36820d..ae9069d2fc 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -4,6 +4,22 @@ #include "clientStmt.h" #include "tdef.h" +int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) { + switch (newStatus) { + case STMT_SETTBNAME: + break; + default: + break; + } + + //STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); + + pStmt->sql.status = newStatus; + + return TSDB_CODE_SUCCESS; +} + + int32_t stmtGetTbName(TAOS_STMT *stmt, char **tbName) { STscStmt* pStmt = (STscStmt*)stmt; @@ -115,6 +131,7 @@ int32_t stmtParseSql(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } + int32_t stmtCleanBindInfo(STscStmt* pStmt) { pStmt->bInfo.tbUid = 0; pStmt->bInfo.tbSuid = 0; @@ -138,7 +155,7 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable) { uint64_t *key = taosHashGetKey(pIter, NULL); if (keepTable && (*key == pStmt->bInfo.tbUid)) { - qResetStmtDataBlock(pBlocks, true); + STMT_ERR_RET(qResetStmtDataBlock(pBlocks, true)); pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter); continue; @@ -187,6 +204,8 @@ int32_t stmtCleanSQLInfo(STscStmt* pStmt) { } int32_t stmtGetFromCache(STscStmt* pStmt) { + pStmt->bInfo.needParse = true; + if (NULL == pStmt->sql.pTableCache || taosHashGetSize(pStmt->sql.pTableCache) <= 0) { return TSDB_CODE_SUCCESS; } @@ -241,6 +260,8 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } + STMT_ERR_RET(stmtCleanBindInfo(pStmt)); + return TSDB_CODE_SUCCESS; } @@ -276,7 +297,11 @@ int stmtPrepare(TAOS_STMT *stmt, const char *sql, unsigned long length) { STMT_ERR_RET(stmtCleanSQLInfo(pStmt)); } - STMT_SWITCH_STATUS(pStmt, STMT_PREPARE, TSDB_CODE_TSC_STMT_API_ERROR); + STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_PREPARE)); + + if (length <= 0) { + length = strlen(sql); + } pStmt->sql.sqlStr = strndup(sql, length); pStmt->sql.sqlLen = length; @@ -288,39 +313,40 @@ int stmtPrepare(TAOS_STMT *stmt, const char *sql, unsigned long length) { int stmtSetTbName(TAOS_STMT *stmt, const char *tbName) { STscStmt* pStmt = (STscStmt*)stmt; - STMT_SWITCH_STATUS(pStmt, STMT_SETTBNAME, TSDB_CODE_TSC_STMT_API_ERROR); - - taosMemoryFree(pStmt->bInfo.tbName); + STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTBNAME)); if (NULL == pStmt->exec.pRequest) { STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest)); } STMT_ERR_RET(qCreateSName(&pStmt->bInfo.sname, tbName, pStmt->taos->acctId, pStmt->exec.pRequest->pDb, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); - - pStmt->bInfo.tbName = strdup(tbName); - + STMT_ERR_RET(stmtGetFromCache(pStmt)); + if (pStmt->bInfo.needParse) { + taosMemoryFree(pStmt->bInfo.tbName); + pStmt->bInfo.tbName = strdup(tbName); + } + return TSDB_CODE_SUCCESS; } int stmtSetTbTags(TAOS_STMT *stmt, TAOS_BIND_v2 *tags) { STscStmt* pStmt = (STscStmt*)stmt; - STMT_SWITCH_STATUS(pStmt, STMT_SETTBNAME, TSDB_CODE_TSC_STMT_API_ERROR); + STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTAGS)); if (pStmt->bInfo.needParse) { STMT_ERR_RET(stmtParseSql(pStmt)); } - STableDataBlocks *pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid)); + STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid)); if (NULL == pDataBlock) { tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid); STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - STMT_ERR_RET(qBindStmtTagsValue(pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.tbSuid, &pStmt->bInfo.sname, tags, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); + STMT_ERR_RET(qBindStmtTagsValue(*pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.tbSuid, &pStmt->bInfo.sname, tags, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); return TSDB_CODE_SUCCESS; } @@ -329,7 +355,7 @@ int stmtSetTbTags(TAOS_STMT *stmt, TAOS_BIND_v2 *tags) { int32_t stmtFetchTagFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fields) { STscStmt* pStmt = (STscStmt*)stmt; - STMT_SWITCH_STATUS(pStmt, STMT_FETCH_TAG_FIELDS, TSDB_CODE_TSC_STMT_API_ERROR); + STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_TAG_FIELDS)); if (pStmt->bInfo.needParse) { STMT_ERR_RET(stmtParseSql(pStmt)); @@ -340,13 +366,13 @@ int32_t stmtFetchTagFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fiel STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); } - STableDataBlocks *pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid)); + STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid)); if (NULL == pDataBlock) { tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid); STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - STMT_ERR_RET(qBuildStmtTagFields(pDataBlock, pStmt->bInfo.boundTags, fieldNum, fields)); + STMT_ERR_RET(qBuildStmtTagFields(*pDataBlock, pStmt->bInfo.boundTags, fieldNum, fields)); return TSDB_CODE_SUCCESS; } @@ -354,7 +380,7 @@ int32_t stmtFetchTagFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fiel int32_t stmtFetchColFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fields) { STscStmt* pStmt = (STscStmt*)stmt; - STMT_SWITCH_STATUS(pStmt, STMT_FETCH_COL_FIELDS, TSDB_CODE_TSC_STMT_API_ERROR); + STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_COL_FIELDS)); if (pStmt->bInfo.needParse) { STMT_ERR_RET(stmtParseSql(pStmt)); @@ -365,13 +391,13 @@ int32_t stmtFetchColFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fiel STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); } - STableDataBlocks *pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid)); + STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid)); if (NULL == pDataBlock) { tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid); STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - STMT_ERR_RET(qBuildStmtColFields(pDataBlock, fieldNum, fields)); + STMT_ERR_RET(qBuildStmtColFields(*pDataBlock, fieldNum, fields)); return TSDB_CODE_SUCCESS; } @@ -379,7 +405,7 @@ int32_t stmtFetchColFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fiel int stmtBindBatch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) { STscStmt* pStmt = (STscStmt*)stmt; - STMT_SWITCH_STATUS(pStmt, STMT_BIND, TSDB_CODE_TSC_STMT_API_ERROR); + STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_BIND)); if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { pStmt->bInfo.needParse = false; @@ -393,13 +419,13 @@ int stmtBindBatch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) { STMT_ERR_RET(stmtParseSql(pStmt)); } - STableDataBlocks *pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid)); + STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid)); if (NULL == pDataBlock) { tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid); STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - qBindStmtColsValue(pDataBlock, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen); + qBindStmtColsValue(*pDataBlock, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen); return TSDB_CODE_SUCCESS; } @@ -408,7 +434,7 @@ int stmtBindBatch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) { int stmtAddBatch(TAOS_STMT *stmt) { STscStmt* pStmt = (STscStmt*)stmt; - STMT_SWITCH_STATUS(pStmt, STMT_ADD_BATCH, TSDB_CODE_TSC_STMT_API_ERROR); + STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_ADD_BATCH)); STMT_ERR_RET(stmtCacheBlock(pStmt)); @@ -419,7 +445,7 @@ int stmtExec(TAOS_STMT *stmt) { STscStmt* pStmt = (STscStmt*)stmt; int32_t code = 0; - STMT_SWITCH_STATUS(pStmt, STMT_EXECUTE, TSDB_CODE_TSC_STMT_API_ERROR); + STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_EXECUTE)); STMT_ERR_RET(qBuildStmtOutput(pStmt->sql.pQuery, pStmt->exec.pVgHash, pStmt->exec.pBlockHash)); @@ -448,6 +474,10 @@ const char *stmtErrstr(TAOS_STMT *stmt) { return (char*) tstrerror(terrno); } + if (pStmt->exec.pRequest) { + pStmt->exec.pRequest->code = terrno; + } + return taos_errstr(pStmt->exec.pRequest); } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 68bea22ea3..08043178d2 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -885,7 +885,7 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken) return TSDB_CODE_SUCCESS; } -static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, int16_t timePrec, int32_t* len, char* tmpTokenBuf) { +static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, int16_t timePrec, bool* gotRow, char* tmpTokenBuf) { SParsedDataColInfo* spd = &pDataBlocks->boundColumnInfo; SRowBuilder* pBuilder = &pDataBlocks->rowBuilder; STSRow* row = (STSRow*)(pDataBlocks->pData + pDataBlocks->size); // skip the SSubmitBlk header @@ -937,6 +937,8 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, } } } + + *gotRow = true; } // *len = pBuilder->extendedRowSize; @@ -967,19 +969,23 @@ static int32_t parseValues(SInsertParseContext* pCxt, STableDataBlocks* pDataBlo maxRows = tSize; } - int32_t len = 0; - CHECK_CODE(parseOneRow(pCxt, pDataBlock, tinfo.precision, &len, tmpTokenBuf)); - pDataBlock->size += extendedRowSize; //len; + bool gotRow = false; + CHECK_CODE(parseOneRow(pCxt, pDataBlock, tinfo.precision, &gotRow, tmpTokenBuf)); + if (gotRow) { + pDataBlock->size += extendedRowSize; //len; + } NEXT_TOKEN(pCxt->pSql, sToken); if (TK_NK_RP != sToken.type) { return buildSyntaxErrMsg(&pCxt->msg, ") expected", sToken.z); } - (*numOfRows)++; + if (gotRow) { + (*numOfRows)++; + } } - if (0 == (*numOfRows)) { + if (0 == (*numOfRows) && (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT))) { return buildSyntaxErrMsg(&pCxt->msg, "no any data points", NULL); } return TSDB_CODE_SUCCESS; @@ -1050,8 +1056,6 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { // for each table while (1) { - destroyInsertParseContextForTable(pCxt); - SToken sToken; char *tbName = NULL; @@ -1060,7 +1064,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { // no data in the sql string anymore. if (sToken.n == 0) { - if (0 == pCxt->totalNum) { + if (0 == pCxt->totalNum && (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT))) { return buildInvalidOperationMsg(&pCxt->msg, "no data in sql");; } break; @@ -1070,6 +1074,8 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { return buildInvalidOperationMsg(&pCxt->msg, "single table allowed in one stmt");; } + destroyInsertParseContextForTable(pCxt); + if (TK_NK_QUESTION == sToken.type) { if (pCxt->pStmtCb) { CHECK_CODE((*pCxt->pStmtCb->getTbNameFn)(pCxt->pStmtCb->pStmt, &tbName)); @@ -1105,7 +1111,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { if (TK_VALUES == sToken.type) { // pSql -> (field1_value, ...) [(field1_value2, ...) ...] CHECK_CODE(parseValuesClause(pCxt, dataBuf)); - pCxt->pOutput->insertType = TSDB_QUERY_TYPE_INSERT; + TSDB_QUERY_SET_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_INSERT); tbNum++; continue; diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index a7481109fa..bf30915fcb 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -572,14 +572,20 @@ int initRowBuilder(SRowBuilder *pBuilder, int16_t schemaVer, SParsedDataColInfo } -void qResetStmtDataBlock(void* block, bool freeData) { +int32_t qResetStmtDataBlock(void* block, bool keepBuf) { STableDataBlocks* pBlock = (STableDataBlocks*)block; - if (freeData) { - taosMemoryFree(pBlock->pData); + if (keepBuf) { + taosMemoryFreeClear(pBlock->pData); + pBlock->pData = taosMemoryMalloc(TSDB_PAYLOAD_SIZE); + if (NULL == pBlock->pData) { + return TSDB_CODE_OUT_OF_MEMORY; + } + memset(pBlock->pData, 0, sizeof(SSubmitBlk)); + } else { + pBlock->pData = NULL; } - - pBlock->pData = NULL; + pBlock->ordered = true; pBlock->prevTS = INT64_MIN; pBlock->size = sizeof(SSubmitBlk); @@ -589,6 +595,8 @@ void qResetStmtDataBlock(void* block, bool freeData) { pBlock->headerSize = pBlock->size; memset(&pBlock->rowBuilder, 0, sizeof(pBlock->rowBuilder)); + + return TSDB_CODE_SUCCESS; } @@ -601,9 +609,7 @@ int32_t qCloneStmtDataBlock(void** pDst, void* pSrc) { memcpy(*pDst, pSrc, sizeof(STableDataBlocks)); ((STableDataBlocks*)(*pDst))->cloned = true; - qResetStmtDataBlock(*pDst, false); - - return TSDB_CODE_SUCCESS; + return qResetStmtDataBlock(*pDst, false); } int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc) { @@ -619,6 +625,8 @@ int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc) { return TSDB_CODE_OUT_OF_MEMORY; } + memset(pBlock->pData, 0, sizeof(SSubmitBlk)); + return TSDB_CODE_SUCCESS; } diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 17ef2076ec..7c67d8967c 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -19,6 +19,10 @@ void taosMsleep(int mseconds); int32_t taosGetTimeOfDay(struct timeval *tv) { return gettimeofday(tv, NULL); } +void *taosMemoryMalloc(int32_t size) { + return malloc(size); +} + void *taosMemoryCalloc(int32_t num, int32_t size) { return calloc(num, size); } @@ -34,6 +38,51 @@ static int64_t taosGetTimestampUs() { return (int64_t)systemTime.tv_sec * 1000000L + (int64_t)systemTime.tv_usec; } + +int stmt_allcol_func1(TAOS_STMT *stmt) { + struct { + int64_t ts; + int32_t v4; + } v = {0}; + int32_t len[10] = {sizeof(v.ts), sizeof(v.v4)}; + + TAOS_BIND_v2 params[10]; + params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[0].buffer_length = sizeof(v.ts); + params[0].buffer = &v.ts; + params[0].length = &len[0]; + params[0].is_null = NULL; + params[0].num = 1; + + params[1].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[1].buffer_length = sizeof(v.v4); + params[1].buffer = &v.v4; + params[1].length = &len[1]; + params[1].is_null = NULL; + params[1].num = 1; + + char *sql = "insert into m0 values(?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + } + + + v.ts = 1591060628000; + v.v4 = 111; + + taos_stmt_bind_param(stmt, params); + taos_stmt_add_batch(stmt); + + if (taos_stmt_execute(stmt) != 0) { + printf("failed to execute insert statement, error:%s.\n", taos_stmt_errstr(stmt)); + exit(1); + } + + return 0; +} + + int stmt_scol_func1(TAOS_STMT *stmt) { struct { int64_t ts; @@ -95,7 +144,7 @@ int stmt_scol_func1(TAOS_STMT *stmt) { char *sql = "insert into ? (ts, v1,v2,f4,bin,bin2) values(?,?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); } for (int zz = 0; zz < 10; zz++) { @@ -103,7 +152,7 @@ int stmt_scol_func1(TAOS_STMT *stmt) { sprintf(buf, "m%d", zz); code = taos_stmt_set_tbname(stmt, buf); if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); + printf("failed to execute taos_stmt_set_tbname. error:%s\n", taos_stmt_errstr(stmt)); exit(1); } v.ts = 1591060628000 + zz * 10; @@ -127,14 +176,13 @@ int stmt_scol_func1(TAOS_STMT *stmt) { } if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); + printf("failed to execute insert statement, error:%s.\n", taos_stmt_errstr(stmt)); exit(1); } return 0; } -#if 0 int stmt_scol_func2(TAOS_STMT *stmt) { struct { @@ -150,42 +198,48 @@ int stmt_scol_func2(TAOS_STMT *stmt) { char blob[80]; } v = {0}; - TAOS_BIND params[10]; + TAOS_BIND_v2 params[10]; params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; params[0].buffer_length = sizeof(v.ts); params[0].buffer = &v.ts; params[0].length = ¶ms[0].buffer_length; params[0].is_null = NULL; + params[0].num = 1; params[1].buffer_type = TSDB_DATA_TYPE_TINYINT; params[1].buffer_length = sizeof(v.v1); params[1].buffer = &v.v1; params[1].length = ¶ms[1].buffer_length; params[1].is_null = NULL; + params[1].num = 1; params[2].buffer_type = TSDB_DATA_TYPE_SMALLINT; params[2].buffer_length = sizeof(v.v2); params[2].buffer = &v.v2; params[2].length = ¶ms[2].buffer_length; params[2].is_null = NULL; + params[2].num = 1; params[3].buffer_type = TSDB_DATA_TYPE_FLOAT; params[3].buffer_length = sizeof(v.f4); params[3].buffer = &v.f4; params[3].length = ¶ms[3].buffer_length; params[3].is_null = NULL; + params[3].num = 1; params[4].buffer_type = TSDB_DATA_TYPE_BINARY; params[4].buffer_length = sizeof(v.bin); params[4].buffer = v.bin; params[4].length = ¶ms[4].buffer_length; params[4].is_null = NULL; + params[4].num = 1; params[5].buffer_type = TSDB_DATA_TYPE_BINARY; params[5].buffer_length = sizeof(v.bin); params[5].buffer = v.bin; params[5].length = ¶ms[5].buffer_length; params[5].is_null = NULL; + params[5].num = 1; char *sql = "insert into m0 (ts, v1,v2,f4,bin,bin2) values(?,?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); @@ -243,7 +297,7 @@ int stmt_scol_func3(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 900000*10); char* is_null = taosMemoryMalloc(sizeof(char) * 60); char* no_null = taosMemoryMalloc(sizeof(char) * 60); @@ -311,7 +365,7 @@ int stmt_scol_func3(TAOS_STMT *stmt) { v.ts[i] = tts + i; } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? (ts, v1,v2,f4,bin,bin2) values(?,?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); @@ -341,7 +395,7 @@ int stmt_scol_func3(TAOS_STMT *stmt) { ++id; } - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); taosMemoryFree(v.ts); @@ -353,6 +407,7 @@ int stmt_scol_func3(TAOS_STMT *stmt) { return 0; } +#if 0 //10 tables 10 records single column bind @@ -4516,11 +4571,33 @@ void* runcase(void *par) { (void)idx; +#if 0 + prepare(taos, 0, 1); + + stmt = taos_stmt_init(taos); + if (NULL == stmt) { + printf("taos_stmt_init failed\n"); + exit(1); + } + + printf("1t+1records start\n"); + stmt_allcol_func1(stmt); + printf("1t+1records end\n"); + printf("check result start\n"); + check_result(taos, "m0", 1, 1); + printf("check result end\n"); + taos_stmt_close(stmt); +#endif -#if 1 + +#if 0 prepare(taos, 1, 1); stmt = taos_stmt_init(taos); + if (NULL == stmt) { + printf("taos_stmt_init failed\n"); + exit(1); + } printf("10t+10records+specifycol start\n"); stmt_scol_func1(stmt); @@ -4540,8 +4617,10 @@ void* runcase(void *par) { taos_stmt_close(stmt); #endif + + + #if 0 -#if 1 prepare(taos, 1, 1); stmt = taos_stmt_init(taos); @@ -4556,6 +4635,7 @@ void* runcase(void *par) { #endif + #if 1 prepare(taos, 1, 1); @@ -4575,6 +4655,8 @@ void* runcase(void *par) { #endif +#if 0 + #if 1 prepare(taos, 1, 1); diff --git a/tests/script/api/makefile b/tests/script/api/makefile index 92d0a89b0f..0c65ac6362 100644 --- a/tests/script/api/makefile +++ b/tests/script/api/makefile @@ -6,16 +6,12 @@ TARGET=exe LFLAGS = '-Wl,-rpath,/usr/local/taos/driver/' -ltaos -lpthread -lm -lrt CFLAGS = -O0 -g -Wall -Wno-deprecated -fPIC -Wno-unused-result -Wconversion \ -Wno-char-subscripts -D_REENTRANT -Wno-format -D_REENTRANT -DLINUX \ - -Wno-unused-function -D_M_X64 -I/usr/local/taos/include -std=gnu99 \ - -fsanitize=address + -Wno-unused-function -D_M_X64 -I/usr/local/taos/include -std=gnu99 all: $(TARGET) exe: gcc $(CFLAGS) ./batchprepare.c -o $(ROOT)batchprepare $(LFLAGS) - gcc $(CFLAGS) ./stmtBatchTest.c -o $(ROOT)stmtBatchTest $(LFLAGS) - gcc $(CFLAGS) ./stmtTest.c -o $(ROOT)stmtTest $(LFLAGS) - gcc $(CFLAGS) ./stmt_function.c -o $(ROOT)stmt_function $(LFLAGS) clean: rm $(ROOT)batchprepare From a1c4f5c2ce8771989b75b7995a61e4bde7fa5d21 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 18 Apr 2022 19:12:10 +0800 Subject: [PATCH 15/47] stmt --- source/client/inc/clientStmt.h | 2 +- source/client/src/clientMain.c | 16 ++++- source/client/src/clientStmt.c | 2 +- tests/script/api/batchprepare.c | 122 ++++++++++++++++---------------- 4 files changed, 77 insertions(+), 65 deletions(-) diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index 07b3e388af..2b9fe4005f 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -100,7 +100,7 @@ int stmtIsInsert(TAOS_STMT *stmt, int *insert); int stmtGetParamNum(TAOS_STMT *stmt, int *nums); int stmtAddBatch(TAOS_STMT *stmt); TAOS_RES *stmtUseResult(TAOS_STMT *stmt); -int stmtBindBatch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind); +int stmtBindBatch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int32_t colIdx); #ifdef __cplusplus diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 8ff5674860..bc26a2c7d4 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -635,7 +635,7 @@ int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) { return terrno; } - return stmtBindBatch(stmt, bind); + return stmtBindBatch(stmt, bind, -1); } int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) { @@ -651,10 +651,22 @@ int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) { return terrno; } - return stmtBindBatch(stmt, bind); + return stmtBindBatch(stmt, bind, -1); } int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int colIdx) { + if (stmt == NULL || bind == NULL) { + tscError("NULL parameter for %s", __FUNCTION__); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + + if (colIdx <= 0) { + tscError("invalid bind column idx %d", colIdx); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + return stmtBindBatch(stmt, bind, colIdx); /* TODO */ } diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index ae9069d2fc..987bf4fa4b 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -402,7 +402,7 @@ int32_t stmtFetchColFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fiel return TSDB_CODE_SUCCESS; } -int stmtBindBatch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) { +int stmtBindBatch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int32_t colIdx) { STscStmt* pStmt = (STscStmt*)stmt; STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_BIND)); diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 7c67d8967c..da291c049e 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -407,7 +407,6 @@ int stmt_scol_func3(TAOS_STMT *stmt) { return 0; } -#if 0 //10 tables 10 records single column bind @@ -428,7 +427,7 @@ int stmt_scol_func4(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1000*10); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 1000*10); char* is_null = taosMemoryMalloc(sizeof(char) * 60); char* no_null = taosMemoryMalloc(sizeof(char) * 60); @@ -488,7 +487,7 @@ int stmt_scol_func4(TAOS_STMT *stmt) { v.ts[i] = tts + i; } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? (ts,b,v4,v8,f8) values(?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); @@ -519,7 +518,7 @@ int stmt_scol_func4(TAOS_STMT *stmt) { } } - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); taosMemoryFree(v.ts); @@ -531,6 +530,7 @@ int stmt_scol_func4(TAOS_STMT *stmt) { return 0; } +#if 0 int stmt_func1(TAOS_STMT *stmt) { @@ -926,7 +926,7 @@ int stmt_funcb_autoctb1(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(10 * sizeof(int)); TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 1*10); // int one_null = 1; int one_not_null = 0; @@ -1075,7 +1075,7 @@ int stmt_funcb_autoctb1(TAOS_STMT *stmt) { } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); @@ -1104,7 +1104,7 @@ int stmt_funcb_autoctb1(TAOS_STMT *stmt) { ++id; - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); taosMemoryFree(v.ts); @@ -1139,7 +1139,7 @@ int stmt_funcb_autoctb2(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(10 * sizeof(int)); TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 1*10); // int one_null = 1; int one_not_null = 0; @@ -1288,7 +1288,7 @@ int stmt_funcb_autoctb2(TAOS_STMT *stmt) { } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? using stb1 tags(1,true,2,3,4,5.0,6.0,'a','b') values(?,?,?,?,?,?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); @@ -1317,7 +1317,7 @@ int stmt_funcb_autoctb2(TAOS_STMT *stmt) { ++id; - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); taosMemoryFree(v.ts); @@ -1353,7 +1353,7 @@ int stmt_funcb_autoctb3(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(10 * sizeof(int)); TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 1*10); // int one_null = 1; int one_not_null = 0; @@ -1477,7 +1477,7 @@ int stmt_funcb_autoctb3(TAOS_STMT *stmt) { } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? using stb1 tags(1,?,2,?,4,?,6.0,?,'b') values(?,?,?,?,?,?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); @@ -1506,7 +1506,7 @@ int stmt_funcb_autoctb3(TAOS_STMT *stmt) { ++id; - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); taosMemoryFree(v.ts); @@ -1544,7 +1544,7 @@ int stmt_funcb_autoctb4(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(10 * sizeof(int)); TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*5); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 1*5); // int one_null = 1; int one_not_null = 0; @@ -1632,7 +1632,7 @@ int stmt_funcb_autoctb4(TAOS_STMT *stmt) { } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? using stb1 tags(1,?,2,?,4,?,6.0,?,'b') (ts,b,v4,v8,f8) values(?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); @@ -1661,7 +1661,7 @@ int stmt_funcb_autoctb4(TAOS_STMT *stmt) { ++id; - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); taosMemoryFree(v.ts); @@ -1697,7 +1697,7 @@ int stmt_funcb_autoctb_e1(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(10 * sizeof(int)); TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 1*10); // int one_null = 1; int one_not_null = 0; @@ -1821,7 +1821,7 @@ int stmt_funcb_autoctb_e1(TAOS_STMT *stmt) { } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? using stb1 (id1,id2,id3,id4,id5,id6,id7,id8,id9) tags(1,?,2,?,4,?,6.0,?,'b') values(?,?,?,?,?,?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); @@ -1850,7 +1850,7 @@ int stmt_funcb_autoctb_e1(TAOS_STMT *stmt) { ++id; - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); taosMemoryFree(v.ts); @@ -1886,7 +1886,7 @@ int stmt_funcb_autoctb_e2(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(10 * sizeof(int)); TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 1*10); // int one_null = 1; int one_not_null = 0; @@ -2035,7 +2035,7 @@ int stmt_funcb_autoctb_e2(TAOS_STMT *stmt) { } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); @@ -2065,7 +2065,7 @@ int stmt_funcb_autoctb_e2(TAOS_STMT *stmt) { ++id; - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); taosMemoryFree(v.ts); @@ -2103,7 +2103,7 @@ int stmt_funcb_autoctb_e3(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(10 * sizeof(int)); TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 1*10); // int one_null = 1; int one_not_null = 0; @@ -2252,7 +2252,7 @@ int stmt_funcb_autoctb_e3(TAOS_STMT *stmt) { } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? using stb1 (id1,id2,id3,id4,id5,id6,id7,id8,id9) tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); @@ -2283,7 +2283,7 @@ int stmt_funcb_autoctb_e3(TAOS_STMT *stmt) { ++id; - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); taosMemoryFree(v.ts); @@ -2318,7 +2318,7 @@ int stmt_funcb_autoctb_e4(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(10 * sizeof(int)); TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 1*10); // int one_null = 1; int one_not_null = 0; @@ -2467,7 +2467,7 @@ int stmt_funcb_autoctb_e4(TAOS_STMT *stmt) { } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); @@ -2508,7 +2508,7 @@ int stmt_funcb_autoctb_e4(TAOS_STMT *stmt) { ++id; - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); taosMemoryFree(v.ts); @@ -2545,7 +2545,7 @@ int stmt_funcb_autoctb_e5(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(10 * sizeof(int)); TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 1*10); // int one_null = 1; int one_not_null = 0; @@ -2694,7 +2694,7 @@ int stmt_funcb_autoctb_e5(TAOS_STMT *stmt) { } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; int code = taos_stmt_prepare(NULL, sql, 0); @@ -2735,7 +2735,7 @@ int stmt_funcb_autoctb_e5(TAOS_STMT *stmt) { ++id; - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); taosMemoryFree(v.ts); @@ -2768,7 +2768,7 @@ int stmt_funcb1(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 900000*10); char* is_null = taosMemoryMalloc(sizeof(char) * 60); char* no_null = taosMemoryMalloc(sizeof(char) * 60); @@ -2864,7 +2864,7 @@ int stmt_funcb1(TAOS_STMT *stmt) { v.ts[i] = tts + i; } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); @@ -2894,7 +2894,7 @@ int stmt_funcb1(TAOS_STMT *stmt) { ++id; } - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); taosMemoryFree(v.ts); @@ -2925,7 +2925,7 @@ int stmt_funcb2(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(18000 * sizeof(int)); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 3000*10); char* is_null = taosMemoryMalloc(sizeof(char) * 18000); char* no_null = taosMemoryMalloc(sizeof(char) * 18000); @@ -3021,7 +3021,7 @@ int stmt_funcb2(TAOS_STMT *stmt) { v.ts[i] = tts + i; } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); @@ -3052,7 +3052,7 @@ int stmt_funcb2(TAOS_STMT *stmt) { } - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); taosMemoryFree(v.ts); @@ -3083,7 +3083,7 @@ int stmt_funcb3(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 900000*10); char* is_null = taosMemoryMalloc(sizeof(char) * 60); char* no_null = taosMemoryMalloc(sizeof(char) * 60); @@ -3185,7 +3185,7 @@ int stmt_funcb3(TAOS_STMT *stmt) { } } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); @@ -3215,7 +3215,7 @@ int stmt_funcb3(TAOS_STMT *stmt) { ++id; } - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); taosMemoryFree(v.ts); @@ -3248,7 +3248,7 @@ int stmt_funcb4(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 900000*10); char* is_null = taosMemoryMalloc(sizeof(char) * 60); char* no_null = taosMemoryMalloc(sizeof(char) * 60); @@ -3344,7 +3344,7 @@ int stmt_funcb4(TAOS_STMT *stmt) { v.ts[i] = tts; } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); @@ -3374,7 +3374,7 @@ int stmt_funcb4(TAOS_STMT *stmt) { ++id; } - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); taosMemoryFree(v.ts); @@ -3407,7 +3407,7 @@ int stmt_funcb5(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(18000 * sizeof(int)); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 3000*10); char* is_null = taosMemoryMalloc(sizeof(char) * 18000); char* no_null = taosMemoryMalloc(sizeof(char) * 18000); @@ -3503,7 +3503,7 @@ int stmt_funcb5(TAOS_STMT *stmt) { v.ts[i] = tts + i; } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into m0 values(?,?,?,?,?,?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); @@ -3527,7 +3527,7 @@ int stmt_funcb5(TAOS_STMT *stmt) { } - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); taosMemoryFree(v.ts); @@ -3551,7 +3551,7 @@ int stmt_funcb_ssz1(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(30000 * sizeof(int)); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 3000*10); char* no_null = taosMemoryMalloc(sizeof(int) * 200000); for (int i = 0; i < 30000; ++i) { @@ -3581,7 +3581,7 @@ int stmt_funcb_ssz1(TAOS_STMT *stmt) { v.ts[i] = tts + i; } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? values(?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); @@ -3612,7 +3612,7 @@ int stmt_funcb_ssz1(TAOS_STMT *stmt) { } - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); taosMemoryFree(v.ts); @@ -3642,7 +3642,7 @@ int stmt_funcb_s1(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 900000*10); char* is_null = taosMemoryMalloc(sizeof(char) * 60); char* no_null = taosMemoryMalloc(sizeof(char) * 60); @@ -3738,7 +3738,7 @@ int stmt_funcb_s1(TAOS_STMT *stmt) { v.ts[i] = tts + i; } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); @@ -3769,7 +3769,7 @@ int stmt_funcb_s1(TAOS_STMT *stmt) { } - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); taosMemoryFree(v.ts); @@ -3804,7 +3804,7 @@ int stmt_funcb_sc1(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 900000*10); char* is_null = taosMemoryMalloc(sizeof(char) * 60); char* no_null = taosMemoryMalloc(sizeof(char) * 60); @@ -3900,7 +3900,7 @@ int stmt_funcb_sc1(TAOS_STMT *stmt) { v.ts[i] = tts + i; } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); @@ -3931,7 +3931,7 @@ int stmt_funcb_sc1(TAOS_STMT *stmt) { } } - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); taosMemoryFree(v.ts); @@ -3962,7 +3962,7 @@ int stmt_funcb_sc2(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 900000*10); char* is_null = taosMemoryMalloc(sizeof(char) * 60); char* no_null = taosMemoryMalloc(sizeof(char) * 60); @@ -4058,7 +4058,7 @@ int stmt_funcb_sc2(TAOS_STMT *stmt) { v.ts[i] = tts + i; } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); @@ -4091,7 +4091,7 @@ int stmt_funcb_sc2(TAOS_STMT *stmt) { } - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); taosMemoryFree(v.ts); @@ -4122,7 +4122,7 @@ int stmt_funcb_sc3(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 60*10); + TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 60*10); char* is_null = taosMemoryMalloc(sizeof(char) * 60); char* no_null = taosMemoryMalloc(sizeof(char) * 60); @@ -4219,7 +4219,7 @@ int stmt_funcb_sc3(TAOS_STMT *stmt) { v.ts[i] = tts + i; } - unsigned long long starttime = getCurrentTime(); + unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); @@ -4248,7 +4248,7 @@ int stmt_funcb_sc3(TAOS_STMT *stmt) { exit(1); } - unsigned long long endtime = getCurrentTime(); + unsigned long long endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); taosMemoryFree(v.ts); From ad299634232e4c07dec38eab35ed6461f250b2bc Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 18 Apr 2022 21:00:35 +0800 Subject: [PATCH 16/47] refactor: modify json compare in where condition --- source/client/src/clientImpl.c | 72 ++++++++++++++++++++++++++- source/libs/scalar/inc/filterInt.h | 2 - source/libs/scalar/src/filter.c | 80 +++++++++--------------------- 3 files changed, 94 insertions(+), 60 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 74b8e711dc..c1351d0fc8 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -619,7 +619,7 @@ void doSetOneRowPtr(SReqResultInfo* pResultInfo) { int32_t bytes = pResultInfo->fields[i].bytes; if (IS_VAR_DATA_TYPE(type)) { - if (pCol->offset[pResultInfo->current] != -1) { + if (!colDataIsNull_var(pCol,pResultInfo->current)) { char* pStart = pResultInfo->pCol[i].offset[pResultInfo->current] + pResultInfo->pCol[i].pData; pResultInfo->length[i] = varDataLen(pStart); @@ -725,6 +725,76 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i]; pResultInfo->row[i] = pResultInfo->pCol[i].pData; } + + if (type == TSDB_DATA_TYPE_JSON) { + char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pResultInfo->convertBuf[i] = p; + int32_t len = 0; + SResultColumn* pCol = &pResultInfo->pCol[i]; + for (int32_t j = 0; j < numOfRows; ++j) { + if (pCol->offset[j] != -1) { + char* pStart = pCol->offset[j] + pCol->pData; + + int32_t jsonInnerType = *pStart; + char *jsonInnerData = pStart + CHAR_BYTES; + char dst[TSDB_MAX_JSON_TAG_LEN] = {0}; + if(jsonInnerType == TSDB_DATA_TYPE_NULL){ + sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L); + varDataSetLen(dst, strlen(varDataVal(dst))); + }else if(jsonInnerType == TSDB_DATA_TYPE_JSON){ + int32_t length = taosUcs4ToMbs((TdUcs4 *)varDataVal(jsonInnerData), varDataLen(jsonInnerData), varDataVal(dst)); + + if (length <= 0) { + tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, varDataVal(jsonInnerData)); + length = 0; + } + varDataSetLen(dst, length); + }else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value" + *(char*)varDataVal(dst) = '\"'; + int32_t length = taosUcs4ToMbs((TdUcs4 *)varDataVal(jsonInnerData), varDataLen(jsonInnerData), varDataVal(dst) + CHAR_BYTES); + if (length <= 0) { + tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, varDataVal(jsonInnerData)); + length = 0; + } + varDataSetLen(dst, length + CHAR_BYTES*2); + *(char*)(varDataVal(dst), length + CHAR_BYTES) = '\"'; + }else if(jsonInnerType == TSDB_DATA_TYPE_DOUBLE){ + double jsonVd = *(double*)(jsonInnerData); + sprintf(varDataVal(dst), "%.9lf", jsonVd); + varDataSetLen(dst, strlen(varDataVal(dst))); + }else if(jsonInnerType == TSDB_DATA_TYPE_BIGINT){ + int64_t jsonVd = *(int64_t*)(jsonInnerData); + sprintf(varDataVal(dst), "%" PRId64, jsonVd); + varDataSetLen(dst, strlen(varDataVal(dst))); + }else if(jsonInnerType == TSDB_DATA_TYPE_BOOL){ + sprintf(varDataVal(dst), "%s", (*((char *)jsonInnerData) == 1) ? "true" : "false"); + varDataSetLen(dst, strlen(varDataVal(dst))); + }else { + ASSERT(0); + } + + if(len + varDataTLen(dst) > colLength[i]){ + p = taosMemoryRealloc(pResultInfo->convertBuf[i], len + varDataTLen(dst)); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pResultInfo->convertBuf[i] = p; + } + p = pResultInfo->convertBuf[i] + len; + memcpy(p, dst, varDataTLen(dst)); + pCol->offset[j] = len; + len += varDataTLen(dst); + } + } + + pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i]; + pResultInfo->row[i] = pResultInfo->pCol[i].pData; + } } return TSDB_CODE_SUCCESS; diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index 977fab2e85..2b9e3c7f39 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -308,7 +308,6 @@ struct SFilterInfo { #define FILTER_GET_COL_FIELD_DATA(fi, ri) (colDataGetData(((SColumnInfoData *)(fi)->data), (ri))) #define FILTER_GET_VAL_FIELD_TYPE(fi) (((SValueNode *)((fi)->desc))->node.resType.type) #define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data) -#define FILTER_GET_JSON_VAL_FIELD_DATA(fi) ((char *)(fi)->desc) #define FILTER_GET_TYPE(fl) ((fl) & FLD_TYPE_MAX) #define FILTER_GROUP_UNIT(i, g, uid) ((i)->units + (g)->unitIdxs[uid]) @@ -321,7 +320,6 @@ struct SFilterInfo { #define FILTER_UNIT_COL_SIZE(i, u) FILTER_GET_COL_FIELD_SIZE(FILTER_UNIT_LEFT_FIELD(i, u)) #define FILTER_UNIT_COL_ID(i, u) FILTER_GET_COL_FIELD_ID(FILTER_UNIT_LEFT_FIELD(i, u)) #define FILTER_UNIT_VAL_DATA(i, u) FILTER_GET_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u)) -#define FILTER_UNIT_JSON_VAL_DATA(i, u) FILTER_GET_JSON_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u)) #define FILTER_UNIT_COL_IDX(u) ((u)->left.idx) #define FILTER_UNIT_OPTR(u) ((u)->compare.optr) #define FILTER_UNIT_COMP_FUNC(u) ((u)->compare.func) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index ea249005b4..ee85b99ad5 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -63,22 +63,18 @@ OptrStr gOptrStr[] = { bool filterRangeCompGi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { int32_t result = cfunc(maxv, minr); - //if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; return result >= 0; } bool filterRangeCompGe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { int32_t result = cfunc(maxv, minr); - //if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; return result > 0; } bool filterRangeCompLi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { int32_t result = cfunc(minv, maxr); - //if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; return result <= 0; } bool filterRangeCompLe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { int32_t result = cfunc(minv, maxr); - //if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; return result < 0; } bool filterRangeCompii (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { @@ -1783,9 +1779,9 @@ int32_t fltInitValFieldData(SFilterInfo *info) { bytes = (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; fi->data = taosMemoryCalloc(1, bytes); - } else if (type != TSDB_DATA_TYPE_JSON){ + } else{ if (dType->type == TSDB_DATA_TYPE_VALUE_ARRAY) { //TIME RANGE -/* +/* fi->data = taosMemoryCalloc(dType->bytes, tDataTypes[type].bytes); for (int32_t a = 0; a < dType->bytes; ++a) { int64_t *v = taosArrayGet(var->arr, a); @@ -1796,31 +1792,27 @@ int32_t fltInitValFieldData(SFilterInfo *info) { } else { fi->data = taosMemoryCalloc(1, sizeof(int64_t)); } - } else{ // type == TSDB_DATA_TYPE_JSON - // fi->data = null; use fi->desc as data, because json value is variable, so use tVariant (fi->desc) } - if(type != TSDB_DATA_TYPE_JSON) { - if (dType->type == type) { - assignVal(fi->data, nodesGetValueFromNode(var), dType->bytes, type); + if (dType->type == type) { + assignVal(fi->data, nodesGetValueFromNode(var), dType->bytes, type); + } else { + SScalarParam out = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))}; + out.columnData->info.type = type; + if (IS_VAR_DATA_TYPE(type)) { + out.columnData->info.bytes = bytes; } else { - SScalarParam out = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))}; - out.columnData->info.type = type; - if (IS_VAR_DATA_TYPE(type)) { - out.columnData->info.bytes = bytes; - } else { - out.columnData->info.bytes = tDataTypes[type].bytes; - } - - // todo refactor the convert - int32_t code = doConvertDataType(var, &out); - if (code != TSDB_CODE_SUCCESS) { - qError("convert value to type[%d] failed", type); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - memcpy(fi->data, out.columnData->pData, out.columnData->info.bytes); + out.columnData->info.bytes = tDataTypes[type].bytes; } + + // todo refactor the convert + int32_t code = doConvertDataType(var, &out); + if (code != TSDB_CODE_SUCCESS) { + qError("convert value to type[%d] failed", type); + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + memcpy(fi->data, out.columnData->pData, out.columnData->info.bytes); } // match/nmatch for nchar type need convert from ucs4 to mbs @@ -2561,11 +2553,7 @@ int32_t filterGenerateComInfo(SFilterInfo *info) { info->cunits[i].colId = FILTER_UNIT_COL_ID(info, unit); if (unit->right.type == FLD_TYPE_VALUE) { - if(FILTER_UNIT_DATA_TYPE(unit) == TSDB_DATA_TYPE_JSON){ // json value is tVariant - info->cunits[i].valData = FILTER_UNIT_JSON_VAL_DATA(info, unit); - }else{ - info->cunits[i].valData = FILTER_UNIT_VAL_DATA(info, unit); - } + info->cunits[i].valData = FILTER_UNIT_VAL_DATA(info, unit); } else { info->cunits[i].valData = NULL; } @@ -2891,18 +2879,8 @@ static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows, for (int32_t i = 0; i < numOfRows; ++i) { uint32_t uidx = info->groups[0].unitIdxs[0]; void *colData = colDataGetData((SColumnInfoData *)info->cunits[uidx].colData, i); - if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){ - if (!colData){ // for json->'key' is null - (*p)[i] = 1; - }else if( *(char*)colData == TSDB_DATA_TYPE_JSON){ // for json is null - colData = POINTER_SHIFT(colData, CHAR_BYTES); - (*p)[i] = colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL); - }else{ - (*p)[i] = 0; - } - }else{ - (*p)[i] = ((colData == NULL) || colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL)); - } + (*p)[i] = ((colData == NULL) || colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL)); + if ((*p)[i] == 0) { all = false; } @@ -2926,19 +2904,7 @@ static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows uint32_t uidx = info->groups[0].unitIdxs[0]; void *colData = colDataGetData((SColumnInfoData *)info->cunits[uidx].colData, i); - if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){ - if (!colData) { // for json->'key' is not null - (*p)[i] = 0; - }else if( *(char*)colData == TSDB_DATA_TYPE_JSON){ // for json is not null - colData = POINTER_SHIFT(colData, CHAR_BYTES); - (*p)[i] = !colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL); - }else{ // for json->'key' is not null - (*p)[i] = 1; - } - }else { - (*p)[i] = ((colData != NULL) && !colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL)); - } - + (*p)[i] = ((colData != NULL) && !colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL)); if ((*p)[i] == 0) { all = false; } From 134a161c7205157157395b3adc23a48be8525d9e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 18 Apr 2022 21:05:34 +0800 Subject: [PATCH 17/47] refactor: modify json compare in where condition --- source/client/src/clientImpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index c1351d0fc8..49f663772d 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -619,7 +619,7 @@ void doSetOneRowPtr(SReqResultInfo* pResultInfo) { int32_t bytes = pResultInfo->fields[i].bytes; if (IS_VAR_DATA_TYPE(type)) { - if (!colDataIsNull_var(pCol,pResultInfo->current)) { + if (pCol->offset[pResultInfo->current] != -1) { char* pStart = pResultInfo->pCol[i].offset[pResultInfo->current] + pResultInfo->pCol[i].pData; pResultInfo->length[i] = varDataLen(pStart); From b96082fe1b36468466e78e76c6ff9017da8f280a Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 19 Apr 2022 09:46:12 +0800 Subject: [PATCH 18/47] refactor: fix compare logic --- source/libs/scalar/src/filter.c | 9 +++++++ source/libs/scalar/src/scalar.c | 1 + source/libs/scalar/src/sclvector.c | 39 +++++++++++++++--------------- 3 files changed, 30 insertions(+), 19 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index ee85b99ad5..23047c2930 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1050,6 +1050,8 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode* tree, SArray *group) { cell = cell->pNext; } + colDataDestroy(out.columnData); + taosMemoryFree(out.columnData); } else { filterAddFieldFromNode(info, node->pRight, &right); @@ -1813,6 +1815,8 @@ int32_t fltInitValFieldData(SFilterInfo *info) { } memcpy(fi->data, out.columnData->pData, out.columnData->info.bytes); + colDataDestroy(out.columnData); + taosMemoryFree(out.columnData); } // match/nmatch for nchar type need convert from ucs4 to mbs @@ -3537,6 +3541,11 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } + if (OP_TYPE_JSON_CONTAINS == node->opType) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } + if (QUERY_NODE_COLUMN != nodeType(node->pLeft)) { SNode *t = node->pLeft; node->pLeft = node->pRight; diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 5f1e08dbcf..9c4d2d94dd 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -132,6 +132,7 @@ void sclFreeRes(SHashObj *res) { void sclFreeParam(SScalarParam *param) { if (param->columnData != NULL) { colDataDestroy(param->columnData); + taosMemoryFree(param->columnData); } if (param->pHashFilter != NULL) { diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index e6deef5824..e97831b1f6 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -188,17 +188,17 @@ static FORCE_INLINE void varToBool(char *buf, SScalarParam* pOut, int32_t rowInd colDataAppendInt8(pOut->columnData, rowIndex, (int8_t*) &v); } -//static FORCE_INLINE void varToNchar(char* buf, SScalarParam* pOut, int32_t rowIndex) { -// int32_t len = 0; -// int32_t inputLen = varDataLen(buf); -// -// char* t = taosMemoryCalloc(1,(inputLen + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); -// /*int32_t resLen = */taosMbsToUcs4(varDataVal(buf), inputLen, (TdUcs4*) varDataVal(t), pOut->columnData->info.bytes, &len); -// varDataSetLen(t, len); -// -// colDataAppend(pOut->columnData, rowIndex, t, false); -// taosMemoryFree(t); -//} +static FORCE_INLINE void varToNchar(char* buf, SScalarParam* pOut, int32_t rowIndex) { + int32_t len = 0; + int32_t inputLen = varDataLen(buf); + + char* t = taosMemoryCalloc(1,(inputLen + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); + /*int32_t resLen = */taosMbsToUcs4(varDataVal(buf), inputLen, (TdUcs4*) varDataVal(t), pOut->columnData->info.bytes, &len); + varDataSetLen(t, len); + + colDataAppend(pOut->columnData, rowIndex, t, false); + taosMemoryFree(t); +} void convertNumberToNumber(const void *inData, void *outData, int8_t inType, int8_t outType){ switch (outType) { @@ -275,7 +275,7 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in int32_t bufSize = pIn->columnData->info.bytes; char *tmp = taosMemoryMalloc(bufSize + VARSTR_HEADER_SIZE); -// bool vton = false; + bool vton = false; _bufConverteFunc func = NULL; if (TSDB_DATA_TYPE_BOOL == outType) { @@ -286,9 +286,10 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in func = varToUnsigned; } else if (IS_FLOAT_TYPE(outType)) { func = varToFloat; -// } else if (outType == TSDB_DATA_TYPE_NCHAR) { // can not be nchar or binary -// func = varToNchar; -// vton = true; + } else if (outType == TSDB_DATA_TYPE_NCHAR) { // binary -> nchar + ASSERT(inType == TSDB_DATA_TYPE_VARCHAR); + func = varToNchar; + vton = true; } else { sclError("invalid convert outType:%d", outType); return TSDB_CODE_QRY_APP_ERROR; @@ -316,9 +317,9 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in continue; } } -// if (vton) { -// memcpy(tmp, data, varDataTLen(data)); -// } else { + if (vton) { + memcpy(tmp, data, varDataTLen(data)); + } else { if (TSDB_DATA_TYPE_VARCHAR == convertType) { memcpy(tmp, varDataVal(data), varDataLen(data)); tmp[varDataLen(data)] = 0; @@ -334,7 +335,7 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in tmp[len] = 0; } -// } + } (*func)(tmp, pOut, i); } From b7bf0fe10bc85243fdfe3e623b0260c3da7bfb4d Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 19 Apr 2022 17:02:49 +0800 Subject: [PATCH 19/47] refactor(tmq): subscribe and rebalance process --- include/common/tmsg.h | 62 ++- include/common/tmsgdef.h | 2 + include/dnode/mnode/sdb/sdb.h | 8 +- include/util/talgo.h | 5 + include/util/taoserror.h | 1 + include/util/tarray.h | 26 +- source/client/src/tmq.c | 9 +- source/common/src/tmsg.c | 8 +- source/dnode/mnode/impl/inc/mndConsumer.h | 17 +- source/dnode/mnode/impl/inc/mndDef.h | 134 +++++- source/dnode/mnode/impl/inc/mndSubscribe.h | 4 +- source/dnode/mnode/impl/src/mndConsumer.c | 536 ++++++++++++++++++++- source/dnode/mnode/impl/src/mndDef.c | 380 +++++++++++++++ source/dnode/mnode/impl/src/mndScheduler.c | 20 +- source/dnode/mnode/impl/src/mndSubscribe.c | 492 ++++++++++++++++++- source/util/src/tarray.c | 41 +- 16 files changed, 1660 insertions(+), 85 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 050a414254..2e0882097c 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -329,7 +329,7 @@ typedef struct SEpSet { int32_t tEncodeSEpSet(SCoder* pEncoder, const SEpSet* pEp); int32_t tDecodeSEpSet(SCoder* pDecoder, SEpSet* pEp); int32_t taosEncodeSEpSet(void** buf, const SEpSet* pEp); -void* taosDecodeSEpSet(void* buf, SEpSet* pEp); +void* taosDecodeSEpSet(const void* buf, SEpSet* pEp); typedef struct { int8_t connType; @@ -1290,10 +1290,14 @@ typedef struct { int32_t tSerializeSCMCreateTopicRsp(void* buf, int32_t bufLen, const SCMCreateTopicRsp* pRsp); int32_t tDeserializeSCMCreateTopicRsp(void* buf, int32_t bufLen, SCMCreateTopicRsp* pRsp); +typedef struct { + int64_t consumerId; +} SMqConsumerLostMsg; + typedef struct { int32_t topicNum; int64_t consumerId; - char* consumerGroup; + char cgroup[TSDB_CGROUP_LEN]; SArray* topicNames; // SArray } SCMSubscribeReq; @@ -1301,7 +1305,7 @@ static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubsc int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pReq->topicNum); tlen += taosEncodeFixedI64(buf, pReq->consumerId); - tlen += taosEncodeString(buf, pReq->consumerGroup); + tlen += taosEncodeString(buf, pReq->cgroup); for (int32_t i = 0; i < pReq->topicNum; i++) { tlen += taosEncodeString(buf, (char*)taosArrayGetP(pReq->topicNames, i)); @@ -1312,7 +1316,7 @@ static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubsc static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq* pReq) { buf = taosDecodeFixedI32(buf, &pReq->topicNum); buf = taosDecodeFixedI64(buf, &pReq->consumerId); - buf = taosDecodeString(buf, &pReq->consumerGroup); + buf = taosDecodeStringTo(buf, pReq->cgroup); pReq->topicNames = taosArrayInit(pReq->topicNum, sizeof(void*)); for (int32_t i = 0; i < pReq->topicNum; i++) { char* name; @@ -1388,10 +1392,10 @@ static FORCE_INLINE void* tDeserializeSMVSubscribeReq(void* buf, SMVSubscribeReq } typedef struct { - const char* key; - SArray* lostConsumers; // SArray - SArray* removedConsumers; // SArray - SArray* newConsumers; // SArray + char key[TSDB_SUBSCRIBE_KEY_LEN]; + SArray* lostConsumers; // SArray + SArray* removedConsumers; // SArray + SArray* newConsumers; // SArray } SMqRebSubscribe; static FORCE_INLINE SMqRebSubscribe* tNewSMqRebSubscribe(const char* key) { @@ -1399,7 +1403,7 @@ static FORCE_INLINE SMqRebSubscribe* tNewSMqRebSubscribe(const char* key) { if (pRebSub == NULL) { goto _err; } - pRebSub->key = strdup(key); + strcpy(pRebSub->key, key); pRebSub->lostConsumers = taosArrayInit(0, sizeof(int64_t)); if (pRebSub->lostConsumers == NULL) { goto _err; @@ -1424,6 +1428,7 @@ _err: // this message is sent from mnode to mnode(read thread to write thread), so there is no need for serialization or // deserialization typedef struct { + int8_t* mqInReb; SHashObj* rebSubHash; // SHashObj } SMqDoRebalanceMsg; @@ -1876,6 +1881,40 @@ static FORCE_INLINE void* taosDecodeSMqMsg(void* buf, SMqHbMsg* pMsg) { return buf; } +typedef struct { + int64_t leftForVer; + int32_t vgId; + int64_t oldConsumerId; + int64_t newConsumerId; + char subKey[TSDB_SUBSCRIBE_KEY_LEN]; + char* qmsg; +} SMqRebVgReq; + +static FORCE_INLINE int32_t tEncodeSMqRebVgReq(void** buf, const SMqRebVgReq* pReq) { + int32_t tlen = 0; + tlen += taosEncodeFixedI64(buf, pReq->leftForVer); + tlen += taosEncodeFixedI32(buf, pReq->vgId); + tlen += taosEncodeFixedI64(buf, pReq->oldConsumerId); + tlen += taosEncodeFixedI64(buf, pReq->newConsumerId); + tlen += taosEncodeString(buf, pReq->subKey); + tlen += taosEncodeString(buf, pReq->qmsg); + return tlen; +} + +static FORCE_INLINE void* tDecodeSMqRebVgReq(const void* buf, SMqRebVgReq* pReq) { + buf = taosDecodeFixedI64(buf, &pReq->leftForVer); + buf = taosDecodeFixedI32(buf, &pReq->vgId); + buf = taosDecodeFixedI64(buf, &pReq->oldConsumerId); + buf = taosDecodeFixedI64(buf, &pReq->newConsumerId); + buf = taosDecodeStringTo(buf, pReq->subKey); + buf = taosDecodeString(buf, &pReq->qmsg); + return (void*)buf; +} + +typedef struct { + int8_t reserved; +} SMqRebVgRsp; + typedef struct { int64_t leftForVer; int32_t vgId; @@ -2441,7 +2480,10 @@ typedef struct { SArray* topics; // SArray } SMqCMGetSubEpRsp; -static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { taosArrayDestroy(pSubTopicEp->vgs); } +static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { + taosMemoryFree(pSubTopicEp->schema.pSchema); + taosArrayDestroy(pSubTopicEp->vgs); +} static FORCE_INLINE int32_t tEncodeSMqSubVgEp(void** buf, const SMqSubVgEp* pVgEp) { int32_t tlen = 0; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index e553dff270..0bc1ea6924 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -149,6 +149,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp) TD_DEF_MSG_TYPE(TDMT_MND_GET_SUB_EP, "mnode-get-sub-ep", SMqCMGetSubEpReq, SMqCMGetSubEpRsp) TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, SMTimerReq) + TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_LOST, "mnode-mq-consumer-lost", SMTimerReq, SMTimerReq) TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, SMqDoRebalanceMsg) TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "mnode-mq-commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp) TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STREAM, "mnode-create-stream", SCMCreateStreamReq, SCMCreateStreamRsp) @@ -180,6 +181,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CONN, "vnode-mq-set-conn", SMqSetCVgReq, SMqSetCVgRsp) TD_DEF_MSG_TYPE(TDMT_VND_MQ_REB, "vnode-mq-mv-rebalance", SMqMVRebReq, SMqMVRebRsp) TD_DEF_MSG_TYPE(TDMT_VND_MQ_CANCEL_CONN, "vnode-mq-mv-cancel-conn", SMqCancelConnReq, SMqCancelConnRsp) + TD_DEF_MSG_TYPE(TDMT_VND_MQ_VG_CHANGE, "vnode-mq-vg-change", SMqRebVgReq, SMqRebVgRsp) TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CUR, "vnode-mq-set-cur", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_RES_READY, "vnode-res-ready", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TASKS_STATUS, "vnode-tasks-status", NULL, NULL) diff --git a/include/dnode/mnode/sdb/sdb.h b/include/dnode/mnode/sdb/sdb.h index a28c093e85..d7d53ad1d0 100644 --- a/include/dnode/mnode/sdb/sdb.h +++ b/include/dnode/mnode/sdb/sdb.h @@ -92,7 +92,13 @@ extern "C" { typedef struct SMnode SMnode; typedef struct SSdbRaw SSdbRaw; typedef struct SSdbRow SSdbRow; -typedef enum { SDB_KEY_BINARY = 1, SDB_KEY_INT32 = 2, SDB_KEY_INT64 = 3 } EKeyType; + +typedef enum { + SDB_KEY_BINARY = 1, + SDB_KEY_INT32 = 2, + SDB_KEY_INT64 = 3, +} EKeyType; + typedef enum { SDB_STATUS_INIT = 0, SDB_STATUS_CREATING = 1, diff --git a/include/util/talgo.h b/include/util/talgo.h index 0fd44a6e91..ebddce62a8 100644 --- a/include/util/talgo.h +++ b/include/util/talgo.h @@ -27,6 +27,11 @@ extern "C" { typedef int32_t (*__compar_fn_t)(const void *, const void *); #endif +typedef void *(*FCopy)(void *); +typedef void (*FDelete)(void *); +typedef int32_t (*FEncode)(void **buf, const void *dst); +typedef void *(*FDecode)(const void *buf, void *dst); + #define TD_EQ 0x1 #define TD_GT 0x2 #define TD_LT 0x4 diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 60535e2f49..fe434fb7f5 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -279,6 +279,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_UNSUPPORTED_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03E8) #define TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E9) #define TSDB_CODE_MND_OFFSET_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03EA) +#define TSDB_CODE_MND_CONSUMER_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x03EB) // mnode-stream #define TSDB_CODE_MND_STREAM_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03F0) diff --git a/include/util/tarray.h b/include/util/tarray.h index 521e54040d..3a97c51c1c 100644 --- a/include/util/tarray.h +++ b/include/util/tarray.h @@ -41,10 +41,10 @@ extern "C" { #define TARRAY_GET_START(array) ((array)->pData) typedef struct SArray { - size_t size; + size_t size; uint32_t capacity; uint32_t elemSize; - void* pData; + void* pData; } SArray; /** @@ -199,24 +199,21 @@ SArray* taosArrayFromList(const void* src, size_t size, size_t elemSize); */ SArray* taosArrayDup(const SArray* pSrc); +/** + * deep copy a new array + * @param pSrc + */ +SArray* taosArrayDeepCopy(const SArray* pSrc, FCopy deepCopy); + /** * clear the array (remove all element) * @param pArray */ void taosArrayClear(SArray* pArray); -/** - * destroy array list - * @param pArray - */ void* taosArrayDestroy(SArray* pArray); - -/** - * - * @param pArray - * @param fp - */ -void taosArrayDestroyEx(SArray* pArray, void (*fp)(void*)); +void taosArrayDestroyP(SArray* pArray, FDelete fp); +void taosArrayDestroyEx(SArray* pArray, FDelete fp); /** * sort the array @@ -264,6 +261,9 @@ char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param); +int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode); +void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz); + #ifdef __cplusplus } #endif diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 478e328a16..9ea7a85d9b 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -255,7 +255,12 @@ int32_t tmq_list_append(tmq_list_t* list, const char* src) { void tmq_list_destroy(tmq_list_t* list) { SArray* container = &list->container; /*taosArrayDestroy(container);*/ - taosArrayDestroyEx(container, (void (*)(void*))taosMemoryFree); + int32_t sz = taosArrayGetSize(container); + for (int32_t i = 0; i < sz; i++) { + char* str = taosArrayGetP(container, i); + taosMemoryFree(str); + } + taosArrayDestroy(container); } static int32_t tmqMakeTopicVgKey(char* dst, const char* topicName, int32_t vg) { @@ -496,7 +501,7 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { SCMSubscribeReq req; req.topicNum = sz; req.consumerId = tmq->consumerId; - req.consumerGroup = strdup(tmq->groupId); + strcpy(req.cgroup, tmq->groupId); req.topicNames = taosArrayInit(sz, sizeof(void*)); for (int i = 0; i < sz; i++) { diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 821f24eedc..e4b1ecaa7e 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -121,14 +121,14 @@ int32_t taosEncodeSEpSet(void **buf, const SEpSet *pEp) { return tlen; } -void *taosDecodeSEpSet(void *buf, SEpSet *pEp) { +void *taosDecodeSEpSet(const void *buf, SEpSet *pEp) { buf = taosDecodeFixedI8(buf, &pEp->inUse); buf = taosDecodeFixedI8(buf, &pEp->numOfEps); for (int32_t i = 0; i < TSDB_MAX_REPLICA; i++) { buf = taosDecodeFixedU16(buf, &pEp->eps[i].port); buf = taosDecodeStringTo(buf, pEp->eps[i].fqdn); } - return buf; + return (void *)buf; } static int32_t tSerializeSClientHbReq(SCoder *pEncoder, const SClientHbReq *pReq) { @@ -2184,7 +2184,7 @@ int32_t tSerializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableReq if (tStartEncode(&encoder) < 0) return -1; if (tEncodeI64(&encoder, pReq->showId) < 0) return -1; if (tEncodeI32(&encoder, pReq->type) < 0) return -1; -// if (tEncodeI8(&encoder, pReq->free) < 0) return -1; + // if (tEncodeI8(&encoder, pReq->free) < 0) return -1; if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; if (tEncodeCStr(&encoder, pReq->tb) < 0) return -1; tEndEncode(&encoder); @@ -2201,7 +2201,7 @@ int32_t tDeserializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableR if (tStartDecode(&decoder) < 0) return -1; if (tDecodeI64(&decoder, &pReq->showId) < 0) return -1; if (tDecodeI32(&decoder, &pReq->type) < 0) return -1; -// if (tDecodeI8(&decoder, &pReq->free) < 0) return -1; + // if (tDecodeI8(&decoder, &pReq->free) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->tb) < 0) return -1; tEndDecode(&decoder); diff --git a/source/dnode/mnode/impl/inc/mndConsumer.h b/source/dnode/mnode/impl/inc/mndConsumer.h index cc345379cf..a818f5d7c3 100644 --- a/source/dnode/mnode/impl/inc/mndConsumer.h +++ b/source/dnode/mnode/impl/inc/mndConsumer.h @@ -22,27 +22,30 @@ extern "C" { #endif - enum { - MQ_CONSUMER_STATUS__INIT = 1, - MQ_CONSUMER_STATUS__IDLE, - MQ_CONSUMER_STATUS__ACTIVE, + // MQ_CONSUMER_STATUS__INIT = 1, + MQ_CONSUMER_STATUS__MODIFY = 1, + MQ_CONSUMER_STATUS__MODIFY_IN_REB, + // MQ_CONSUMER_STATUS__IDLE, + MQ_CONSUMER_STATUS__READY, MQ_CONSUMER_STATUS__LOST, - MQ_CONSUMER_STATUS__MODIFY + MQ_CONSUMER_STATUS__LOST_IN_REB, + MQ_CONSUMER_STATUS__LOST_REBD, }; - int32_t mndInitConsumer(SMnode *pMnode); void mndCleanupConsumer(SMnode *pMnode); SMqConsumerObj *mndAcquireConsumer(SMnode *pMnode, int64_t consumerId); void mndReleaseConsumer(SMnode *pMnode, SMqConsumerObj *pConsumer); -SMqConsumerObj* mndCreateConsumer(int64_t consumerId, const char* cgroup); +SMqConsumerObj *mndCreateConsumer(int64_t consumerId, const char *cgroup); SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer); SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw); +int32_t mndSetConsumerCommitLogs(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index eaec99c186..a96be41308 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -88,6 +88,7 @@ typedef enum { TRN_TYPE_CREATE_STREAM = 1019, TRN_TYPE_DROP_STREAM = 1020, TRN_TYPE_ALTER_STREAM = 1021, + TRN_TYPE_CONSUMER_LOST = 1022, TRN_TYPE_BASIC_SCOPE_END, TRN_TYPE_GLOBAL_SCOPE = 2000, TRN_TYPE_CREATE_DNODE = 2001, @@ -506,6 +507,7 @@ static FORCE_INLINE void* tDecodeSMqOffsetObj(void* buf, SMqOffsetObj* pOffset) return buf; } +#if 0 typedef struct { char key[TSDB_SUBSCRIBE_KEY_LEN]; int32_t status; @@ -636,6 +638,7 @@ static FORCE_INLINE void tDeleteSMqSubscribeObj(SMqSubscribeObj* pSub) { pSub->unassignedVg = NULL; } } +#endif typedef struct { char name[TSDB_TOPIC_FNAME_LEN]; @@ -654,6 +657,7 @@ typedef struct { SSchemaWrapper schema; } SMqTopicObj; +#if 0 typedef struct { int64_t consumerId; int64_t connId; @@ -708,7 +712,7 @@ static FORCE_INLINE void* tDecodeSMqConsumerObj(void* buf, SMqConsumerObj* pCons buf = taosDecodeStringTo(buf, pConsumer->cgroup); buf = taosDecodeFixedI32(buf, &sz); - pConsumer->currentTopics = taosArrayInit(sz, sizeof(SMqConsumerObj)); + pConsumer->currentTopics = taosArrayInit(sz, sizeof(void*)); for (int32_t i = 0; i < sz; i++) { char* topic; buf = taosDecodeString(buf, &topic); @@ -716,7 +720,7 @@ static FORCE_INLINE void* tDecodeSMqConsumerObj(void* buf, SMqConsumerObj* pCons } buf = taosDecodeFixedI32(buf, &sz); - pConsumer->recentRemovedTopics = taosArrayInit(sz, sizeof(SMqConsumerObj)); + pConsumer->recentRemovedTopics = taosArrayInit(sz, sizeof(void*)); for (int32_t i = 0; i < sz; i++) { char* topic; buf = taosDecodeString(buf, &topic); @@ -724,6 +728,132 @@ static FORCE_INLINE void* tDecodeSMqConsumerObj(void* buf, SMqConsumerObj* pCons } return buf; } +#endif + +enum { + CONSUMER_UPDATE__TOUCH = 1, + CONSUMER_UPDATE__ADD, + CONSUMER_UPDATE__REMOVE, + CONSUMER_UPDATE__LOST, + CONSUMER_UPDATE__MODIFY, +}; + +typedef struct { + int64_t consumerId; + char cgroup[TSDB_CGROUP_LEN]; + int8_t updateType; // used only for update + int32_t epoch; + int32_t status; + // hbStatus is not applicable to serialization + int32_t hbStatus; + // lock is used for topics update + SRWLatch lock; + SArray* currentTopics; // SArray +#if 0 + SArray* waitingRebTopics; // SArray +#endif + SArray* rebNewTopics; // SArray + SArray* rebRemovedTopics; // SArray +} SMqConsumerObj; + +SMqConsumerObj* tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_LEN]); +void tDeleteSMqConsumerObj(SMqConsumerObj* pConsumer); +int32_t tEncodeSMqConsumerObj(void** buf, const SMqConsumerObj* pConsumer); +void* tDecodeSMqConsumerObj(const void* buf, SMqConsumerObj* pConsumer); + +typedef struct { + int32_t vgId; + char* qmsg; + // char topic[TSDB_TOPIC_FNAME_LEN]; + SEpSet epSet; +} SMqVgEp; + +SMqVgEp* tCloneSMqVgEp(const SMqVgEp* pVgEp); +void tDeleteSMqVgEp(SMqVgEp* pVgEp); +int32_t tEncodeSMqVgEp(void** buf, const SMqVgEp* pVgEp); +void* tDecodeSMqVgEp(const void* buf, SMqVgEp* pVgEp); + +typedef struct { + int64_t consumerId; // -1 for unassigned + SArray* vgs; // SArray +} SMqConsumerEpInSub; + +SMqConsumerEpInSub* tCloneSMqConsumerEpInSub(const SMqConsumerEpInSub* pEpInSub); +void tDeleteSMqConsumerEpInSub(SMqConsumerEpInSub* pEpInSub); +int32_t tEncodeSMqConsumerEpInSub(void** buf, const SMqConsumerEpInSub* pEpInSub); +void* tDecodeSMqConsumerEpInSub(const void* buf, SMqConsumerEpInSub* pEpInSub); + +typedef struct { + char key[TSDB_SUBSCRIBE_KEY_LEN]; + SRWLatch lock; + int32_t vgNum; + SHashObj* consumerHash; // consumerId -> SMqConsumerEpInSub +} SMqSubscribeObj; + +SMqSubscribeObj* tNewSubscribeObj(const char key[TSDB_SUBSCRIBE_KEY_LEN]); +SMqSubscribeObj* tCloneSubscribeObj(const SMqSubscribeObj* pSub); +void tDeleteSubscribeObj(SMqSubscribeObj* pSub); +int32_t tEncodeSubscribeObj(void** buf, const SMqSubscribeObj* pSub); +void* tDecodeSubscribeObj(const void* buf, SMqSubscribeObj* pSub); + +typedef struct { + int32_t epoch; + SArray* consumers; // SArray +} SMqSubActionLogEntry; + +SMqSubActionLogEntry* tCloneSMqSubActionLogEntry(SMqSubActionLogEntry* pEntry); +void tDeleteSMqSubActionLogEntry(SMqSubActionLogEntry* pEntry); +int32_t tEncodeSMqSubActionLogEntry(void** buf, const SMqSubActionLogEntry* pEntry); +void* tDecodeSMqSubActionLogEntry(const void* buf, SMqSubActionLogEntry* pEntry); + +typedef struct { + char key[TSDB_SUBSCRIBE_KEY_LEN]; + SArray* logs; // SArray +} SMqSubActionLogObj; + +SMqSubActionLogObj* tCloneSMqSubActionLogObj(SMqSubActionLogObj* pLog); +void tDeleteSMqSubActionLogObj(SMqSubActionLogObj* pLog); +int32_t tEncodeSMqSubActionLogObj(void** buf, const SMqSubActionLogObj* pLog); +void* tDecodeSMqSubActionLogObj(const void* buf, SMqSubActionLogObj* pLog); + +typedef struct { + int64_t consumerId; + char cgroup[TSDB_CGROUP_LEN]; + SRWLatch lock; + SArray* vgs; // SArray +} SMqConsumerEpObj; + +SMqConsumerEpObj* tCloneSMqConsumerEpObj(const SMqConsumerEpObj* pConsumerEp); +void tDeleteSMqConsumerEpObj(SMqConsumerEpObj* pConsumerEp); +int32_t tEncodeSMqConsumerEpObj(void** buf, const SMqConsumerEpObj* pConsumerEp); +void* tDecodeSMqConsumerEpObj(const void* buf, SMqConsumerEpObj* pConsumerEp); + +typedef struct { + const SMqSubscribeObj* pOldSub; + const SMqTopicObj* pTopic; + const SMqRebSubscribe* pRebInfo; +} SMqRebInputObj; + +typedef struct { + int64_t oldConsumerId; + int64_t newConsumerId; + SMqVgEp* pVgEp; +} SMqRebOutputVg; + +#if 0 +typedef struct { + int64_t consumerId; +} SMqRebOutputConsumer; +#endif + +typedef struct { + SArray* rebVgs; // SArray + SArray* newConsumers; // SArray + SArray* removedConsumers; // SArray + SArray* touchedConsumers; // SArray + SMqSubscribeObj* pSub; + SMqSubActionLogEntry* pLogEntry; +} SMqRebOutputObj; typedef struct { char name[TSDB_TOPIC_FNAME_LEN]; diff --git a/source/dnode/mnode/impl/inc/mndSubscribe.h b/source/dnode/mnode/impl/inc/mndSubscribe.h index 3f897067a2..7915006f27 100644 --- a/source/dnode/mnode/impl/inc/mndSubscribe.h +++ b/source/dnode/mnode/impl/inc/mndSubscribe.h @@ -26,9 +26,11 @@ int32_t mndInitSubscribe(SMnode *pMnode); void mndCleanupSubscribe(SMnode *pMnode); SMqSubscribeObj *mndAcquireSubscribe(SMnode *pMnode, const char *CGroup, const char *topicName); -SMqSubscribeObj *mndAcquireSubscribeByKey(SMnode *pMnode, const char* key); +SMqSubscribeObj *mndAcquireSubscribeByKey(SMnode *pMnode, const char *key); void mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub); +int32_t mndMakeSubscribeKey(char *key, const char *cgroup, const char *topicName); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index d2a3c38135..8a66e5cb69 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -19,8 +19,10 @@ #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" +#include "mndOffset.h" #include "mndShow.h" #include "mndStb.h" +#include "mndSubscribe.h" #include "mndTopic.h" #include "mndTrans.h" #include "mndUser.h" @@ -28,9 +30,13 @@ #include "tcompare.h" #include "tname.h" -#define MND_CONSUMER_VER_NUMBER 1 +#define MND_CONSUMER_VER_NUMBER 1 #define MND_CONSUMER_RESERVE_SIZE 64 +#define MND_CONSUMER_LOST_HB_CNT 3 + +static int8_t mqInRebFlag = 0; + static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer); static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer); static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pConsumer, SMqConsumerObj *pNewConsumer); @@ -38,6 +44,11 @@ static int32_t mndProcessConsumerMetaMsg(SNodeMsg *pMsg); static int32_t mndRetrieveConsumer(SNodeMsg *pMsg, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter); +static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg); +static int32_t mndProcessAskEpReq(SNodeMsg *pMsg); +static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg); +static int32_t mndProcessConsumerLostMsg(SNodeMsg *pMsg); + int32_t mndInitConsumer(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_CONSUMER, .keyType = SDB_KEY_INT64, @@ -47,25 +58,392 @@ int32_t mndInitConsumer(SMnode *pMnode) { .updateFp = (SdbUpdateFp)mndConsumerActionUpdate, .deleteFp = (SdbDeleteFp)mndConsumerActionDelete}; + mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq); + mndSetMsgHandle(pMnode, TDMT_MND_GET_SUB_EP, mndProcessAskEpReq); + mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg); + mndSetMsgHandle(pMnode, TDMT_MND_MQ_CONSUMER_LOST, mndProcessConsumerLostMsg); return sdbSetTable(pMnode->pSdb, table); } void mndCleanupConsumer(SMnode *pMnode) {} -SMqConsumerObj *mndCreateConsumer(int64_t consumerId, const char *cgroup) { - SMqConsumerObj *pConsumer = taosMemoryCalloc(1, sizeof(SMqConsumerObj)); - if (pConsumer == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; +static int32_t mndProcessConsumerLostMsg(SNodeMsg *pMsg) { + SMnode *pMnode = pMsg->pNode; + SMqConsumerLostMsg *pLostMsg = pMsg->rpcMsg.pCont; + SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pLostMsg->consumerId); + ASSERT(pConsumer); + + SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup); + pConsumerNew->updateType = CONSUMER_UPDATE__LOST; + + mndReleaseConsumer(pMnode, pConsumer); + + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CONSUMER_LOST, &pMsg->rpcMsg); + if (pTrans == NULL) goto FAIL; + if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto FAIL; + if (mndTransPrepare(pMnode, pTrans) != 0) goto FAIL; + + mndTransDrop(pTrans); + return 0; +FAIL: + // TODO delete consumer + mndTransDrop(pTrans); + return -1; +} + +static SMqRebSubscribe *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { + SMqRebSubscribe *pRebSub = taosHashGet(pHash, key, strlen(key) + 1); + if (pRebSub == NULL) { + pRebSub = tNewSMqRebSubscribe(key); + if (pRebSub == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + taosHashPut(pHash, key, strlen(key) + 1, pRebSub, sizeof(SMqRebSubscribe)); + } + return pRebSub; +} + +static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) { + SMnode *pMnode = pMsg->pNode; + SSdb *pSdb = pMnode->pSdb; + SMqConsumerObj *pConsumer; + void *pIter = NULL; + + // rebalance cannot be parallel + int8_t old = atomic_val_compare_exchange_8(&mqInRebFlag, 0, 1); + if (old != 0) { + mInfo("mq rebalance already in progress, do nothing"); + return 0; } - pConsumer->recentRemovedTopics = taosArrayInit(1, sizeof(char *)); - pConsumer->epoch = 1; - pConsumer->consumerId = consumerId; - atomic_store_32(&pConsumer->status, MQ_CONSUMER_STATUS__INIT); - strcpy(pConsumer->cgroup, cgroup); - taosInitRWLatch(&pConsumer->lock); - return pConsumer; + SMqDoRebalanceMsg *pRebMsg = rpcMallocCont(sizeof(SMqDoRebalanceMsg)); + pRebMsg->rebSubHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK); + // TODO set cleanfp + pRebMsg->mqInReb = &mqInRebFlag; + + // iterate all consumers, find all modification + while (1) { + pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer); + if (pIter == NULL) break; + + taosRLockLatch(&pConsumer->lock); + int32_t hbStatus = atomic_add_fetch_32(&pConsumer->hbStatus, 1); + int32_t status = atomic_load_32(&pConsumer->status); + if (status == MQ_CONSUMER_STATUS__READY && hbStatus > MND_CONSUMER_LOST_HB_CNT) { + SMqConsumerLostMsg *pLostMsg = rpcMallocCont(sizeof(SMqConsumerLostMsg)); + + pLostMsg->consumerId = pConsumer->consumerId; + SRpcMsg *pRpcMsg = taosMemoryCalloc(1, sizeof(SRpcMsg)); + pRpcMsg->msgType = TDMT_MND_MQ_CONSUMER_LOST; + pRpcMsg->pCont = pLostMsg; + pRpcMsg->contLen = sizeof(SMqConsumerLostMsg); + tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, pRpcMsg); + } + if (status == MQ_CONSUMER_STATUS__LOST_REBD || status == MQ_CONSUMER_STATUS__READY) { + // do nothing + } else if (status == MQ_CONSUMER_STATUS__LOST) { + int32_t topicNum = taosArrayGetSize(pConsumer->currentTopics); + for (int32_t i = 0; i < topicNum; i++) { + char key[TSDB_SUBSCRIBE_KEY_LEN]; + char *removedTopic = taosArrayGetP(pConsumer->currentTopics, i); + mndMakeSubscribeKey(key, pConsumer->cgroup, removedTopic); + SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); + taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId); + } + } else if (status == MQ_CONSUMER_STATUS__MODIFY) { + int32_t newTopicNum = taosArrayGetSize(pConsumer->rebNewTopics); + for (int32_t i = 0; i < newTopicNum; i++) { + char key[TSDB_SUBSCRIBE_KEY_LEN]; + char *newTopic = taosArrayGetP(pConsumer->rebNewTopics, i); + mndMakeSubscribeKey(key, pConsumer->cgroup, newTopic); + SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); + taosArrayPush(pRebSub->newConsumers, &pConsumer->consumerId); + } + + int32_t removedTopicNum = taosArrayGetSize(pConsumer->rebRemovedTopics); + for (int32_t i = 0; i < removedTopicNum; i++) { + char key[TSDB_SUBSCRIBE_KEY_LEN]; + char *removedTopic = taosArrayGetP(pConsumer->rebRemovedTopics, i); + mndMakeSubscribeKey(key, pConsumer->cgroup, removedTopic); + SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); + taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId); + } + } else { + // do nothing + } + + taosRUnLockLatch(&pConsumer->lock); + mndReleaseConsumer(pMnode, pConsumer); + } + + if (taosHashGetSize(pRebMsg->rebSubHash) != 0) { + mInfo("mq rebalance will be triggered"); + SRpcMsg rpcMsg = { + .msgType = TDMT_MND_MQ_DO_REBALANCE, + .pCont = pRebMsg, + .contLen = sizeof(SMqDoRebalanceMsg), + }; + tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); + } else { + taosHashCleanup(pRebMsg->rebSubHash); + rpcFreeCont(pRebMsg); + mInfo("mq rebalance finished, no modification"); + atomic_store_8(&mqInRebFlag, 0); + } + return 0; +} + +static int32_t mndProcessAskEpReq(SNodeMsg *pMsg) { + SMnode *pMnode = pMsg->pNode; + SMqCMGetSubEpReq *pReq = (SMqCMGetSubEpReq *)pMsg->rpcMsg.pCont; + SMqCMGetSubEpRsp rsp = {0}; + int64_t consumerId = be64toh(pReq->consumerId); + int32_t epoch = ntohl(pReq->epoch); + + SMqConsumerObj *pConsumer = mndAcquireConsumer(pMsg->pNode, consumerId); + if (pConsumer == NULL) { + terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST; + return -1; + } + + ASSERT(strcmp(pReq->cgroup, pConsumer->cgroup) == 0); + /*int32_t hbStatus = atomic_load_32(&pConsumer->hbStatus);*/ + atomic_store_32(&pConsumer->hbStatus, 0); + + // 1. check consumer status + int32_t status = atomic_load_32(&pConsumer->status); + if (status == MQ_CONSUMER_STATUS__LOST) { + // recover consumer + } + + if (status != MQ_CONSUMER_STATUS__READY) { + mndReleaseConsumer(pMnode, pConsumer); + terrno = TSDB_CODE_MND_CONSUMER_NOT_READY; + return -1; + } + + int32_t serverEpoch = atomic_load_32(&pConsumer->epoch); + + // 2. check epoch, only send ep info when epoches do not match + if (epoch != serverEpoch) { + taosRLockLatch(&pConsumer->lock); + int32_t numOfTopics = taosArrayGetSize(pConsumer->currentTopics); + + rsp.topics = taosArrayInit(numOfTopics, sizeof(SMqSubTopicEp)); + if (rsp.topics == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto FAIL; + } + + // handle all topic subscribed by the consumer + for (int32_t i = 0; i < numOfTopics; i++) { + char *topic = taosArrayGetP(pConsumer->currentTopics, i); + SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, topic); + + // txn guarantees pSub is created + ASSERT(pSub); + taosRLockLatch(&pSub->lock); + + SMqSubTopicEp topicEp = {0}; + strcpy(topicEp.topic, topic); + + // 2.1 fetch topic schema + SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); + ASSERT(pTopic); + taosRLockLatch(&pTopic->lock); + topicEp.schema.nCols = pTopic->schema.nCols; + topicEp.schema.pSchema = taosMemoryCalloc(topicEp.schema.nCols, sizeof(SSchema)); + memcpy(topicEp.schema.pSchema, pTopic->schema.pSchema, topicEp.schema.nCols * sizeof(SSchema)); + taosRUnLockLatch(&pTopic->lock); + mndReleaseTopic(pMnode, pTopic); + + // 2.2 iterate all vg assigned to the consumer of that topic + SMqConsumerEpInSub *pEpInSub = taosHashGet(pSub->consumerHash, &consumerId, sizeof(int64_t)); + int32_t vgNum = taosArrayGetSize(pEpInSub->vgs); + + topicEp.vgs = taosArrayInit(vgNum, sizeof(SMqSubVgEp)); + if (topicEp.vgs == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto FAIL; + } + + for (int32_t j = 0; j < vgNum; j++) { + SMqVgEp *pVgEp = taosArrayGetP(pEpInSub->vgs, j); + char offsetKey[TSDB_PARTITION_KEY_LEN]; + mndMakePartitionKey(offsetKey, pConsumer->cgroup, topic, pVgEp->vgId); + // 2.2.1 build vg ep + SMqSubVgEp vgEp = { + .epSet = pVgEp->epSet, + .vgId = pVgEp->vgId, + .offset = -1, + }; + + // 2.2.2 fetch vg offset + SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, offsetKey); + if (pOffsetObj != NULL) { + vgEp.offset = atomic_load_64(&pOffsetObj->offset); + mndReleaseOffset(pMnode, pOffsetObj); + } + taosArrayPush(topicEp.vgs, &vgEp); + } + taosArrayPush(rsp.topics, &topicEp); + + taosRUnLockLatch(&pSub->lock); + mndReleaseSubscribe(pMnode, pSub); + } + taosRUnLockLatch(&pConsumer->lock); + } + // encode rsp + int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqCMGetSubEpRsp(NULL, &rsp); + void *buf = rpcMallocCont(tlen); + if (buf == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + ((SMqRspHead *)buf)->mqMsgType = TMQ_MSG_TYPE__EP_RSP; + ((SMqRspHead *)buf)->epoch = serverEpoch; + ((SMqRspHead *)buf)->consumerId = pConsumer->consumerId; + + void *abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); + tEncodeSMqCMGetSubEpRsp(&abuf, &rsp); + + // release consumer and free memory + tDeleteSMqCMGetSubEpRsp(&rsp); + mndReleaseConsumer(pMnode, pConsumer); + + // send rsp + pMsg->pRsp = buf; + pMsg->rspLen = tlen; + return 0; +FAIL: + tDeleteSMqCMGetSubEpRsp(&rsp); + mndReleaseConsumer(pMnode, pConsumer); + return -1; +} + +int32_t mndSetConsumerCommitLogs(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer) { + SSdbRaw *pCommitRaw = mndConsumerActionEncode(pConsumer); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; + return 0; +} + +static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) { + SMnode *pMnode = pMsg->pNode; + char *msgStr = pMsg->rpcMsg.pCont; + SCMSubscribeReq subscribe = {0}; + tDeserializeSCMSubscribeReq(msgStr, &subscribe); + int64_t consumerId = subscribe.consumerId; + char *cgroup = subscribe.cgroup; + SMqConsumerObj *pConsumerOld = NULL; + SMqConsumerObj *pConsumerNew = NULL; + + int32_t code = -1; + SArray *newSub = subscribe.topicNames; + taosArraySortString(newSub, taosArrayCompareString); + + int32_t newTopicNum = taosArrayGetSize(newSub); + // check topic existance + for (int32_t i = 0; i < newTopicNum; i++) { + char *topic = taosArrayGetP(newSub, i); + SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); + if (pTopic == NULL) { + terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST; + goto SUBSCRIBE_OVER; + } + // TODO lock topic to prevent drop + mndReleaseTopic(pMnode, pTopic); + } + + pConsumerOld = mndAcquireConsumer(pMnode, consumerId); + if (pConsumerOld == NULL) { + pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup); + pConsumerNew->updateType = CONSUMER_UPDATE__MODIFY; + /*pConsumerNew->waitingRebTopics = newSub;*/ + pConsumerNew->rebNewTopics = newSub; + subscribe.topicNames = NULL; + + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_SUBSCRIBE, &pMsg->rpcMsg); + if (pTrans == NULL) goto SUBSCRIBE_OVER; + if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto SUBSCRIBE_OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto SUBSCRIBE_OVER; + + } else { + taosRLockLatch(&pConsumerOld->lock); + int32_t status = atomic_load_32(&pConsumerOld->status); + if (status != MQ_CONSUMER_STATUS__READY) { + terrno = TSDB_CODE_MND_CONSUMER_NOT_READY; + goto SUBSCRIBE_OVER; + } + + pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup); + if (pConsumerNew == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto SUBSCRIBE_OVER; + } + pConsumerNew->updateType = CONSUMER_UPDATE__MODIFY; + /*pConsumerOld->waitingRebTopics = newSub;*/ + + int32_t oldTopicNum = 0; + if (pConsumerOld->currentTopics) { + oldTopicNum = taosArrayGetSize(pConsumerOld->currentTopics); + } + + int32_t i = 0, j = 0; + while (i < oldTopicNum || j < newTopicNum) { + if (i >= oldTopicNum) { + char *newTopicCopy = strdup(taosArrayGetP(newSub, j)); + taosArrayPush(pConsumerNew->rebNewTopics, &newTopicCopy); + j++; + continue; + } else if (j >= newTopicNum) { + char *oldTopicCopy = strdup(taosArrayGetP(pConsumerOld->currentTopics, i)); + taosArrayPush(pConsumerNew->rebRemovedTopics, &oldTopicCopy); + i++; + continue; + } else { + char *oldTopic = taosArrayGetP(pConsumerOld->currentTopics, i); + char *newTopic = taosArrayGetP(newSub, j); + int comp = compareLenPrefixedStr(oldTopic, newTopic); + if (comp == 0) { + i++; + j++; + continue; + } else if (comp < 0) { + char *oldTopicCopy = strdup(oldTopic); + taosArrayPush(pConsumerNew->rebRemovedTopics, &oldTopicCopy); + i++; + continue; + } else { + char *newTopicCopy = strdup(newTopic); + taosArrayPush(pConsumerNew->rebNewTopics, &newTopicCopy); + j++; + continue; + } + } + } + + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_SUBSCRIBE, &pMsg->rpcMsg); + if (pTrans == NULL) goto SUBSCRIBE_OVER; + if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto SUBSCRIBE_OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto SUBSCRIBE_OVER; + } + + code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + +SUBSCRIBE_OVER: + if (pConsumerOld) { + taosRUnLockLatch(&pConsumerOld->lock); + mndReleaseConsumer(pMnode, pConsumerOld); + } + if (pConsumerNew) { + tDeleteSMqConsumerObj(pConsumerNew); + } + // TODO: replace with destroy subscribe msg + if (subscribe.topicNames) taosArrayDestroyP(subscribe.topicNames, (FDelete)taosMemoryFree); + return code; } SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) { @@ -154,15 +532,141 @@ static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer) { static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer) { mTrace("consumer:%" PRId64 ", perform delete action", pConsumer->consumerId); + tDeleteSMqConsumerObj(pConsumer); return 0; } static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, SMqConsumerObj *pNewConsumer) { mTrace("consumer:%" PRId64 ", perform update action", pOldConsumer->consumerId); - /*taosWLockLatch(&pOldConsumer->lock);*/ - atomic_add_fetch_32(&pOldConsumer->epoch, 1); - /*taosWUnLockLatch(&pOldConsumer->lock);*/ + taosWLockLatch(&pOldConsumer->lock); + + if (pNewConsumer->updateType == CONSUMER_UPDATE__MODIFY) { + ASSERT(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0); + ASSERT(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0); + SArray *tmp = pOldConsumer->rebNewTopics; + pOldConsumer->rebNewTopics = pNewConsumer->rebNewTopics; + pNewConsumer->rebNewTopics = tmp; + + tmp = pOldConsumer->rebRemovedTopics; + pOldConsumer->rebRemovedTopics = pNewConsumer->rebRemovedTopics; + pNewConsumer->rebRemovedTopics = tmp; + + pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY; + } else if (pNewConsumer->updateType == CONSUMER_UPDATE__LOST) { + int32_t sz = taosArrayGetSize(pOldConsumer->currentTopics); + pOldConsumer->rebRemovedTopics = taosArrayInit(sz, sizeof(void *)); + for (int32_t i = 0; i < sz; i++) { + char *topic = strdup(taosArrayGetP(pOldConsumer->currentTopics, i)); + taosArrayPush(pNewConsumer->rebRemovedTopics, &topic); + } + pOldConsumer->status = MQ_CONSUMER_STATUS__LOST; + } else if (pNewConsumer->updateType == CONSUMER_UPDATE__TOUCH) { + atomic_add_fetch_32(&pOldConsumer->epoch, 1); + } else if (pNewConsumer->updateType == CONSUMER_UPDATE__ADD) { + ASSERT(taosArrayGetSize(pNewConsumer->rebNewTopics) == 1); + ASSERT(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0); + + char *addedTopic = strdup(taosArrayGetP(pNewConsumer->rebNewTopics, 0)); + // not exist in current topic +#if 1 + for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->currentTopics); i++) { + char *topic = taosArrayGetP(pOldConsumer->currentTopics, i); + ASSERT(strcmp(topic, addedTopic) != 0); + } +#endif + + // remove from new topic + for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->rebNewTopics); i++) { + char *topic = taosArrayGetP(pOldConsumer->rebNewTopics, i); + if (strcmp(addedTopic, topic) == 0) { + taosArrayRemove(pOldConsumer->rebNewTopics, i); + taosMemoryFree(topic); + break; + } + } + + // add to current topic + taosArrayPush(pOldConsumer->currentTopics, &addedTopic); + taosArraySortString(pOldConsumer->currentTopics, taosArrayCompareString); + // set status + if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0) { + if (pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY || + pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) { + pOldConsumer->status = MQ_CONSUMER_STATUS__READY; + } else if (pOldConsumer->status == MQ_CONSUMER_STATUS__LOST_IN_REB || + pOldConsumer->status == MQ_CONSUMER_STATUS__LOST) { + pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_REBD; + } + } else { + if (pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY || + pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) { + pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY_IN_REB; + } else if (pOldConsumer->status == MQ_CONSUMER_STATUS__LOST || + pOldConsumer->status == MQ_CONSUMER_STATUS__LOST_IN_REB) { + pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_IN_REB; + } + } + atomic_add_fetch_32(&pOldConsumer->epoch, 1); + } else if (pNewConsumer->updateType == CONSUMER_UPDATE__REMOVE) { + ASSERT(taosArrayGetSize(pNewConsumer->rebNewTopics) == 0); + ASSERT(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 1); + char *removedTopic = taosArrayGetP(pNewConsumer->rebRemovedTopics, 0); + + // not exist in new topic +#if 1 + for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->rebNewTopics); i++) { + char *topic = taosArrayGetP(pOldConsumer->rebNewTopics, i); + ASSERT(strcmp(topic, removedTopic) != 0); + } +#endif + + // remove from removed topic + for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->rebRemovedTopics); i++) { + char *topic = taosArrayGetP(pOldConsumer->rebNewTopics, i); + if (strcmp(removedTopic, topic) == 0) { + taosArrayRemove(pOldConsumer->rebNewTopics, i); + taosMemoryFree(topic); + break; + } + } + + // remove from current topic + int32_t i = 0; + int32_t sz = taosArrayGetSize(pOldConsumer->currentTopics); + for (i = 0; i < sz; i++) { + char *topic = taosArrayGetP(pOldConsumer->currentTopics, i); + if (strcmp(removedTopic, topic) == 0) { + taosArrayRemove(pOldConsumer->currentTopics, i); + taosMemoryFree(topic); + break; + } + } + // must find the topic + ASSERT(i < sz); + + // set status + if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0) { + if (pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY || + pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) { + pOldConsumer->status = MQ_CONSUMER_STATUS__READY; + } else if (pOldConsumer->status == MQ_CONSUMER_STATUS__LOST_IN_REB || + pOldConsumer->status == MQ_CONSUMER_STATUS__LOST) { + pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_REBD; + } + } else { + if (pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY || + pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) { + pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY_IN_REB; + } else if (pOldConsumer->status == MQ_CONSUMER_STATUS__LOST || + pOldConsumer->status == MQ_CONSUMER_STATUS__LOST_IN_REB) { + pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_IN_REB; + } + } + atomic_add_fetch_32(&pOldConsumer->epoch, 1); + } + + taosWUnLockLatch(&pOldConsumer->lock); return 0; } diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 6374b4cad2..dd2d6801e1 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -14,6 +14,386 @@ */ #include "mndDef.h" +#include "mndConsumer.h" + +SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_LEN]) { + SMqConsumerObj *pConsumer = taosMemoryCalloc(1, sizeof(SMqConsumerObj)); + if (pConsumer == NULL) { + return NULL; + } + + pConsumer->consumerId = consumerId; + memcpy(pConsumer->cgroup, cgroup, TSDB_CGROUP_LEN); + + pConsumer->epoch = 0; + pConsumer->status = MQ_CONSUMER_STATUS__MODIFY; + pConsumer->hbStatus = 0; + + taosInitRWLatch(&pConsumer->lock); + + pConsumer->currentTopics = taosArrayInit(0, sizeof(void *)); +#if 0 + pConsumer->waitingRebTopics = NULL; +#endif + pConsumer->rebNewTopics = taosArrayInit(0, sizeof(void *)); + pConsumer->rebRemovedTopics = taosArrayInit(0, sizeof(void *)); + + if (pConsumer->currentTopics == NULL || pConsumer->rebNewTopics == NULL || pConsumer->rebRemovedTopics == NULL) { + taosArrayDestroy(pConsumer->currentTopics); + taosArrayDestroy(pConsumer->rebNewTopics); + taosArrayDestroy(pConsumer->rebRemovedTopics); + taosMemoryFree(pConsumer); + return NULL; + } + + return pConsumer; +} + +void tDeleteSMqConsumerObj(SMqConsumerObj *pConsumer) { + if (pConsumer->currentTopics) { + taosArrayDestroyP(pConsumer->currentTopics, (FDelete)taosMemoryFree); + } +#if 0 + if (pConsumer->waitingRebTopics) { + taosArrayDestroyP(pConsumer->waitingRebTopics, taosMemoryFree); + } +#endif + if (pConsumer->rebNewTopics) { + taosArrayDestroyP(pConsumer->rebNewTopics, (FDelete)taosMemoryFree); + } + if (pConsumer->rebRemovedTopics) { + taosArrayDestroyP(pConsumer->rebRemovedTopics, (FDelete)taosMemoryFree); + } +} + +int32_t tEncodeSMqConsumerObj(void **buf, const SMqConsumerObj *pConsumer) { + int32_t tlen = 0; + int32_t sz; + tlen += taosEncodeFixedI64(buf, pConsumer->consumerId); + tlen += taosEncodeString(buf, pConsumer->cgroup); + tlen += taosEncodeFixedI8(buf, pConsumer->updateType); + tlen += taosEncodeFixedI32(buf, pConsumer->epoch); + tlen += taosEncodeFixedI32(buf, pConsumer->status); + + // current topics + if (pConsumer->currentTopics) { + sz = taosArrayGetSize(pConsumer->currentTopics); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + char *topic = taosArrayGetP(pConsumer->currentTopics, i); + tlen += taosEncodeString(buf, topic); + } + } else { + tlen += taosEncodeFixedI32(buf, 0); + } + +#if 0 + // waiting reb topics + if (pConsumer->waitingRebTopics) { + sz = taosArrayGetSize(pConsumer->waitingRebTopics); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + char *topic = taosArrayGetP(pConsumer->waitingRebTopics, i); + tlen += taosEncodeString(buf, topic); + } + } else { + tlen += taosEncodeFixedI32(buf, 0); + } +#endif + + // reb new topics + if (pConsumer->rebNewTopics) { + sz = taosArrayGetSize(pConsumer->rebNewTopics); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + char *topic = taosArrayGetP(pConsumer->rebNewTopics, i); + tlen += taosEncodeString(buf, topic); + } + } else { + tlen += taosEncodeFixedI32(buf, 0); + } + + // reb removed topics + if (pConsumer->rebRemovedTopics) { + sz = taosArrayGetSize(pConsumer->rebRemovedTopics); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + char *topic = taosArrayGetP(pConsumer->rebRemovedTopics, i); + tlen += taosEncodeString(buf, topic); + } + } else { + tlen += taosEncodeFixedI32(buf, 0); + } + + return tlen; +} + +void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer) { + int32_t sz; + buf = taosDecodeFixedI64(buf, &pConsumer->consumerId); + buf = taosDecodeStringTo(buf, pConsumer->cgroup); + buf = taosDecodeFixedI8(buf, &pConsumer->updateType); + buf = taosDecodeFixedI32(buf, &pConsumer->epoch); + buf = taosDecodeFixedI32(buf, &pConsumer->status); + + // current topics + buf = taosDecodeFixedI32(buf, &sz); + pConsumer->currentTopics = taosArrayInit(sz, sizeof(void *)); + for (int32_t i = 0; i < sz; i++) { + char *topic; + buf = taosDecodeString(buf, &topic); + taosArrayPush(pConsumer->currentTopics, &topic); + } + +#if 0 + // waiting reb topics + buf = taosDecodeFixedI32(buf, &sz); + pConsumer->waitingRebTopics = taosArrayInit(sz, sizeof(void *)); + for (int32_t i = 0; i < sz; i++) { + char *topic; + buf = taosDecodeString(buf, &topic); + taosArrayPush(pConsumer->waitingRebTopics, &topic); + } +#endif + + // reb new topics + buf = taosDecodeFixedI32(buf, &sz); + pConsumer->rebNewTopics = taosArrayInit(sz, sizeof(void *)); + for (int32_t i = 0; i < sz; i++) { + char *topic; + buf = taosDecodeString(buf, &topic); + taosArrayPush(pConsumer->rebNewTopics, &topic); + } + + // reb removed topics + buf = taosDecodeFixedI32(buf, &sz); + pConsumer->rebRemovedTopics = taosArrayInit(sz, sizeof(void *)); + for (int32_t i = 0; i < sz; i++) { + char *topic; + buf = taosDecodeString(buf, &topic); + taosArrayPush(pConsumer->rebRemovedTopics, &topic); + } + + return (void *)buf; +} + +SMqVgEp *tCloneSMqVgEp(const SMqVgEp *pVgEp) { + SMqVgEp *pVgEpNew = taosMemoryMalloc(sizeof(SMqVgEp)); + if (pVgEpNew == NULL) return NULL; + pVgEpNew->vgId = pVgEp->vgId; + pVgEpNew->qmsg = strdup(pVgEp->qmsg); + /*memcpy(pVgEpNew->topic, pVgEp->topic, TSDB_TOPIC_FNAME_LEN);*/ + pVgEpNew->epSet = pVgEp->epSet; + return pVgEpNew; +} + +void tDeleteSMqVgEp(SMqVgEp *pVgEp) { taosMemoryFree(pVgEp->qmsg); } + +int32_t tEncodeSMqVgEp(void **buf, const SMqVgEp *pVgEp) { + int32_t tlen = 0; + tlen += taosEncodeFixedI32(buf, pVgEp->vgId); + tlen += taosEncodeString(buf, pVgEp->qmsg); + /*tlen += taosEncodeString(buf, pVgEp->topic);*/ + tlen += taosEncodeSEpSet(buf, &pVgEp->epSet); + return tlen; +} + +void *tDecodeSMqVgEp(const void *buf, SMqVgEp *pVgEp) { + buf = taosDecodeFixedI32(buf, &pVgEp->vgId); + buf = taosDecodeString(buf, &pVgEp->qmsg); + /*buf = taosDecodeStringTo(buf, pVgEp->topic);*/ + buf = taosDecodeSEpSet(buf, &pVgEp->epSet); + return (void *)buf; +} + +SMqConsumerEpObj *tCloneSMqConsumerEpObj(const SMqConsumerEpObj *pConsumerEp) { + SMqConsumerEpObj *pConsumerEpNew = taosMemoryMalloc(sizeof(SMqConsumerEpObj)); + if (pConsumerEpNew == NULL) return NULL; + pConsumerEpNew->consumerId = pConsumerEp->consumerId; + memcpy(pConsumerEpNew->cgroup, pConsumerEp->cgroup, TSDB_CGROUP_LEN); + taosInitRWLatch(&pConsumerEpNew->lock); + pConsumerEpNew->vgs = taosArrayDeepCopy(pConsumerEpNew->vgs, (FCopy)tCloneSMqVgEp); + return pConsumerEpNew; +} + +void tDeleteSMqConsumerEpObj(SMqConsumerEpObj *pConsumerEp) { + taosArrayDestroyEx(pConsumerEp->vgs, (FDelete)tDeleteSMqVgEp); +} + +int32_t tEncodeSMqConsumerEpObj(void **buf, const SMqConsumerEpObj *pConsumerEp) { + int32_t tlen = 0; + tlen += taosEncodeFixedI64(buf, pConsumerEp->consumerId); + tlen += taosEncodeString(buf, pConsumerEp->cgroup); + tlen += taosEncodeArray(buf, pConsumerEp->vgs, (FEncode)tEncodeSMqVgEp); + return tlen; +} + +void *tDecodeSMqConsumerEpObj(const void *buf, SMqConsumerEpObj *pConsumerEp) { + buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId); + buf = taosDecodeStringTo(buf, pConsumerEp->cgroup); + buf = taosDecodeArray(buf, &pConsumerEp->vgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqSubVgEp)); + return (void *)buf; +} + +SMqConsumerEpInSub *tCloneSMqConsumerEpInSub(const SMqConsumerEpInSub *pEpInSub) { + SMqConsumerEpInSub *pEpInSubNew = taosMemoryMalloc(sizeof(SMqConsumerEpInSub)); + if (pEpInSubNew == NULL) return NULL; + pEpInSubNew->consumerId = pEpInSub->consumerId; + pEpInSubNew->vgs = taosArrayDeepCopy(pEpInSub->vgs, (FCopy)tCloneSMqVgEp); + return pEpInSubNew; +} + +void tDeleteSMqConsumerEpInSub(SMqConsumerEpInSub *pEpInSub) { + taosArrayDestroyEx(pEpInSub->vgs, (FDelete)tDeleteSMqVgEp); +} + +int32_t tEncodeSMqConsumerEpInSub(void **buf, const SMqConsumerEpInSub *pEpInSub) { + int32_t tlen = 0; + tlen += taosEncodeFixedI64(buf, pEpInSub->consumerId); + tlen += taosEncodeArray(buf, pEpInSub->vgs, (FEncode)tEncodeSMqVgEp); + return tlen; +} + +void *tDecodeSMqConsumerEpInSub(const void *buf, SMqConsumerEpInSub *pEpInSub) { + buf = taosDecodeFixedI64(buf, &pEpInSub->consumerId); + buf = taosDecodeArray(buf, &pEpInSub->vgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqSubVgEp)); + return (void *)buf; +} + +SMqSubscribeObj *tNewSubscribeObj(const char key[TSDB_SUBSCRIBE_KEY_LEN]) { + SMqSubscribeObj *pSubNew = taosMemoryMalloc(sizeof(SMqSubscribeObj)); + if (pSubNew == NULL) return NULL; + memcpy(pSubNew->key, key, TSDB_SUBSCRIBE_KEY_LEN); + taosInitRWLatch(&pSubNew->lock); + pSubNew->vgNum = -1; + pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + // TODO set free fp + SMqConsumerEpInSub *pEpInSub = taosMemoryMalloc(sizeof(SMqConsumerEpInSub)); + pEpInSub->vgs = taosArrayInit(0, sizeof(SMqVgEp)); + int64_t unexistKey = -1; + taosHashPut(pSubNew->consumerHash, &unexistKey, sizeof(int64_t), pEpInSub, sizeof(SMqConsumerEpInSub)); + return pSubNew; +} + +SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) { + SMqSubscribeObj *pSubNew = taosMemoryMalloc(sizeof(SMqSubscribeObj)); + if (pSubNew == NULL) return NULL; + memcpy(pSubNew->key, pSub->key, TSDB_SUBSCRIBE_KEY_LEN); + taosInitRWLatch(&pSubNew->lock); + pSubNew->vgNum = pSub->vgNum; + /*pSubNew->consumerEps = taosArrayDeepCopy(pSub->consumerEps, (FCopy)tCloneSMqConsumerEpInSub);*/ + pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + /*taosHashSetFreeFp(pSubNew->consumerHash, taosArrayDestroy);*/ + void *pIter = NULL; + SMqConsumerEpInSub *pEpInSub = NULL; + while (1) { + pIter = taosHashIterate(pSubNew->consumerHash, pIter); + if (pIter == NULL) break; + pEpInSub = (SMqConsumerEpInSub *)pIter; + SMqConsumerEpInSub newEp = { + .consumerId = pEpInSub->consumerId, + .vgs = taosArrayDeepCopy(pEpInSub->vgs, (FCopy)tCloneSMqVgEp), + }; + taosHashPut(pSubNew->consumerHash, &newEp.consumerId, sizeof(int64_t), &newEp, sizeof(SMqConsumerEpInSub)); + } + return pSubNew; +} + +void tDeleteSubscribeObj(SMqSubscribeObj *pSub) { + /*taosArrayDestroyEx(pSub->consumerEps, (FDelete)tDeleteSMqConsumerEpInSub);*/ + taosHashCleanup(pSub->consumerHash); +} + +int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) { + int32_t tlen = 0; + tlen += taosEncodeString(buf, pSub->key); + tlen += taosEncodeFixedI32(buf, pSub->vgNum); + + void *pIter = NULL; + int32_t sz = taosHashGetSize(pSub->consumerHash); + tlen += taosEncodeFixedI32(buf, sz); + + int32_t cnt = 0; + while (1) { + pIter = taosHashIterate(pSub->consumerHash, pIter); + if (pIter == NULL) break; + SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter; + tlen += tEncodeSMqConsumerEpInSub(buf, pEpInSub); + cnt++; + } + ASSERT(cnt == sz); + /*tlen += taosEncodeArray(buf, pSub->consumerEps, (FEncode)tEncodeSMqConsumerEpInSub);*/ + return tlen; +} + +void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub) { + // + buf = taosDecodeStringTo(buf, pSub->key); + buf = taosDecodeFixedI32(buf, &pSub->vgNum); + + int32_t sz; + buf = taosDecodeFixedI32(buf, &sz); + + pSub->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + for (int32_t i = 0; i < sz; i++) { + /*SMqConsumerEpInSub* pEpInSub = taosMemoryMalloc(sizeof(SMqConsumerEpInSub));*/ + SMqConsumerEpInSub epInSub = {0}; + buf = tDecodeSMqConsumerEpInSub(buf, &epInSub); + taosHashPut(pSub->consumerHash, &epInSub.consumerId, sizeof(int64_t), &epInSub, sizeof(SMqConsumerEpInSub)); + } + + /*buf = taosDecodeArray(buf, &pSub->consumerEps, (FDecode)tDecodeSMqConsumerEpInSub, sizeof(SMqConsumerEpInSub));*/ + return (void *)buf; +} + +SMqSubActionLogEntry *tCloneSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) { + SMqSubActionLogEntry *pEntryNew = taosMemoryMalloc(sizeof(SMqSubActionLogEntry)); + if (pEntryNew == NULL) return NULL; + pEntryNew->epoch = pEntry->epoch; + pEntryNew->consumers = taosArrayDeepCopy(pEntry->consumers, (FCopy)tCloneSMqConsumerEpInSub); + return pEntryNew; +} + +void tDeleteSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) { + taosArrayDestroyEx(pEntry->consumers, (FDelete)tDeleteSMqConsumerEpInSub); +} + +int32_t tEncodeSMqSubActionLogEntry(void **buf, const SMqSubActionLogEntry *pEntry) { + int32_t tlen = 0; + tlen += taosEncodeFixedI32(buf, pEntry->epoch); + tlen += taosEncodeArray(buf, pEntry->consumers, (FEncode)tEncodeSMqSubActionLogEntry); + return tlen; +} +void *tDecodeSMqSubActionLogEntry(const void *buf, SMqSubActionLogEntry *pEntry) { + buf = taosDecodeFixedI32(buf, &pEntry->epoch); + buf = taosDecodeArray(buf, &pEntry->consumers, (FDecode)tDecodeSMqSubActionLogEntry, sizeof(SMqSubActionLogEntry)); + return (void *)buf; +} + +SMqSubActionLogObj *tCloneSMqSubActionLogObj(SMqSubActionLogObj *pLog) { + SMqSubActionLogObj *pLogNew = taosMemoryMalloc(sizeof(SMqSubActionLogObj)); + if (pLogNew == NULL) return pLogNew; + memcpy(pLogNew->key, pLog->key, TSDB_SUBSCRIBE_KEY_LEN); + pLogNew->logs = taosArrayDeepCopy(pLog->logs, (FCopy)tCloneSMqConsumerEpInSub); + return pLogNew; +} + +void tDeleteSMqSubActionLogObj(SMqSubActionLogObj *pLog) { + taosArrayDestroyEx(pLog->logs, (FDelete)tDeleteSMqConsumerEpInSub); +} + +int32_t tEncodeSMqSubActionLogObj(void **buf, const SMqSubActionLogObj *pLog) { + int32_t tlen = 0; + tlen += taosEncodeString(buf, pLog->key); + tlen += taosEncodeArray(buf, pLog->logs, (FEncode)tEncodeSMqSubActionLogEntry); + return tlen; +} + +void *tDecodeSMqSubActionLogObj(const void *buf, SMqSubActionLogObj *pLog) { + buf = taosDecodeStringTo(buf, pLog->key); + buf = taosDecodeArray(buf, &pLog->logs, (FDecode)tDecodeSMqSubActionLogEntry, sizeof(SMqSubActionLogEntry)); + return (void *)buf; +} int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) { int32_t sz = 0; diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index df7946a0c1..5629421a84 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -453,6 +453,9 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib } SSubplan* plan = nodesListGetNode(inner->pNodeList, 0); + int64_t unexistKey = -1; + SMqConsumerEpInSub* pEpInSub = taosHashGet(pSub->consumerHash, &unexistKey, sizeof(int64_t)); + void* pIter = NULL; while (1) { pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); @@ -466,24 +469,33 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib plan->execNode.nodeId = pVgroup->vgId; plan->execNode.epSet = mndGetVgroupEpset(pMnode, pVgroup); + SMqVgEp* pVgEp = taosMemoryMalloc(sizeof(SMqVgEp)); + pVgEp->epSet = plan->execNode.epSet; + pVgEp->vgId = plan->execNode.nodeId; + +#if 0 SMqConsumerEp consumerEp = {0}; consumerEp.status = 0; consumerEp.consumerId = -1; consumerEp.epSet = plan->execNode.epSet; consumerEp.vgId = plan->execNode.nodeId; - - mDebug("init subscribption %s, assign vg: %d", pSub->key, consumerEp.vgId); +#endif + + mDebug("init subscribption %s, assign vg: %d", pSub->key, pVgEp->vgId); int32_t msgLen; - if (qSubPlanToString(plan, &consumerEp.qmsg, &msgLen) < 0) { + if (qSubPlanToString(plan, &pVgEp->qmsg, &msgLen) < 0) { sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } - taosArrayPush(pSub->unassignedVg, &consumerEp); + taosArrayPush(pEpInSub->vgs, &pVgEp); + + /*taosArrayPush(pSub->unassignedVg, &consumerEp);*/ } + taosHashRelease(pSub->consumerHash, pEpInSub); qDestroyQueryPlan(pPlan); return 0; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 3a31b0abb9..48c9804619 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -40,30 +40,47 @@ enum { MQ_SUBSCRIBE_STATUS__DELETED, }; -static int32_t mndMakeSubscribeKey(char *key, const char *cgroup, const char *topicName); - static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *); static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw); static int32_t mndSubActionInsert(SSdb *pSdb, SMqSubscribeObj *); static int32_t mndSubActionDelete(SSdb *pSdb, SMqSubscribeObj *); static int32_t mndSubActionUpdate(SSdb *pSdb, SMqSubscribeObj *pOldSub, SMqSubscribeObj *pNewSub); -static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg); -static int32_t mndProcessSubscribeRsp(SNodeMsg *pMsg); +/*static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg);*/ +/*static int32_t mndProcessSubscribeRsp(SNodeMsg *pMsg);*/ static int32_t mndProcessSubscribeInternalReq(SNodeMsg *pMsg); static int32_t mndProcessSubscribeInternalRsp(SNodeMsg *pMsg); -static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg); -static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg); -static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg); -static int32_t mndProcessResetOffsetReq(SNodeMsg *pMsg); +/*static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg);*/ +/*static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg);*/ +/*static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg);*/ +/*static int32_t mndProcessResetOffsetReq(SNodeMsg *pMsg);*/ -static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup, - const SMqConsumerEp *pConsumerEp); +static int32_t mndProcessRebalanceReq(SNodeMsg *pMsg); -static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp, - const char *topicName); -static int32_t mndPersistCancelConnReq(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp, - const char *oldTopicName); +static int32_t mndSetSubRedoLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub) { + SSdbRaw *pRedoRaw = mndSubActionEncode(pSub); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) return -1; + return 0; +} + +static int32_t mndSetSubCommitLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub) { + SSdbRaw *pCommitRaw = mndSubActionEncode(pSub); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; + return 0; +} + +/*static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup,*/ +/*const SMqConsumerEp *pConsumerEp);*/ + +/*static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp,*/ +/*const char *topicName);*/ + +/*static int32_t mndPersistCancelConnReq(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp,*/ +/*const char *oldTopicName);*/ int32_t mndInitSubscribe(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_SUBSCRIBE, @@ -74,16 +91,35 @@ int32_t mndInitSubscribe(SMnode *pMnode) { .updateFp = (SdbUpdateFp)mndSubActionUpdate, .deleteFp = (SdbDeleteFp)mndSubActionDelete}; - mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq); - mndSetMsgHandle(pMnode, TDMT_VND_MQ_SET_CONN_RSP, mndProcessSubscribeInternalRsp); - mndSetMsgHandle(pMnode, TDMT_VND_MQ_REB_RSP, mndProcessSubscribeInternalRsp); - mndSetMsgHandle(pMnode, TDMT_VND_MQ_CANCEL_CONN_RSP, mndProcessSubscribeInternalRsp); - mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg); - mndSetMsgHandle(pMnode, TDMT_MND_GET_SUB_EP, mndProcessGetSubEpReq); - mndSetMsgHandle(pMnode, TDMT_MND_MQ_DO_REBALANCE, mndProcessDoRebalanceMsg); + /*mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq);*/ + /*mndSetMsgHandle(pMnode, TDMT_VND_MQ_SET_CONN_RSP, mndProcessSubscribeInternalRsp);*/ + /*mndSetMsgHandle(pMnode, TDMT_VND_MQ_REB_RSP, mndProcessSubscribeInternalRsp);*/ + /*mndSetMsgHandle(pMnode, TDMT_VND_MQ_CANCEL_CONN_RSP, mndProcessSubscribeInternalRsp);*/ + /*mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg);*/ + /*mndSetMsgHandle(pMnode, TDMT_MND_GET_SUB_EP, mndProcessGetSubEpReq);*/ + /*mndSetMsgHandle(pMnode, TDMT_MND_MQ_DO_REBALANCE, mndProcessDoRebalanceMsg);*/ + mndSetMsgHandle(pMnode, TDMT_VND_MQ_VG_CHANGE_RSP, mndProcessSubscribeInternalRsp); + mndSetMsgHandle(pMnode, TDMT_MND_MQ_DO_REBALANCE, mndProcessRebalanceReq); return sdbSetTable(pMnode->pSdb, table); } +static SMqSubscribeObj *mndCreateSub(SMnode *pMnode, const SMqTopicObj *pTopic, const char *subKey) { + SMqSubscribeObj *pSub = tNewSubscribeObj(subKey); + if (pSub == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + if (mndSchedInitSubEp(pMnode, pTopic, pSub) < 0) { + tDeleteSubscribeObj(pSub); + taosMemoryFree(pSub); + return NULL; + } + + return pSub; +} + +#if 0 static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj *pTopic, const char *cgroup) { SMqSubscribeObj *pSub = tNewSubscribeObj(); if (pSub == NULL) { @@ -212,7 +248,63 @@ static int32_t mndPersistCancelConnReq(SMnode *pMnode, STrans *pTrans, const SMq return 0; } +#endif +static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const char *subKey, const SMqRebOutputVg *pRebVg) { + SMqRebVgReq req = {0}; + req.oldConsumerId = pRebVg->oldConsumerId; + req.newConsumerId = pRebVg->newConsumerId; + req.vgId = pRebVg->pVgEp->vgId; + req.qmsg = req.qmsg; + strncpy(req.subKey, subKey, TSDB_SUBSCRIBE_KEY_LEN); + + int32_t tlen = sizeof(SMsgHead) + tEncodeSMqRebVgReq(NULL, &req); + void *buf = taosMemoryMalloc(tlen); + if (buf == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + SMsgHead *pMsgHead = (SMsgHead *)buf; + + pMsgHead->contLen = htonl(tlen); + pMsgHead->vgId = htonl(pRebVg->pVgEp->vgId); + + void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + tEncodeSMqRebVgReq(&abuf, &req); + *pBuf = buf; + *pLen = tlen; + + return 0; +} + +static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, const char *subKey, + const SMqRebOutputVg *pRebVg) { + ASSERT(pRebVg->oldConsumerId != pRebVg->newConsumerId); + + void *buf; + int32_t tlen; + if (mndBuildSubChangeReq(&buf, &tlen, subKey, pRebVg) < 0) { + return -1; + } + + int32_t vgId = pRebVg->pVgEp->vgId; + SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId); + + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgObj); + action.pCont = buf; + action.contLen = tlen; + action.msgType = TDMT_VND_MQ_VG_CHANGE; + + mndReleaseVgroup(pMnode, pVgObj); + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(buf); + return -1; + } + return 0; +} + +#if 0 static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) { SMnode *pMnode = pMsg->pNode; SMqCMGetSubEpReq *pReq = (SMqCMGetSubEpReq *)pMsg->rpcMsg.pCont; @@ -312,6 +404,7 @@ static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) { pMsg->rspLen = tlen; return 0; } +#endif static int32_t mndSplitSubscribeKey(const char *key, char *topic, char *cgroup) { int32_t i = 0; @@ -337,6 +430,7 @@ static SMqRebSubscribe *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { return pRebSub; } +#if 0 static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) { SMnode *pMnode = pMsg->pNode; SSdb *pSdb = pMnode->pSdb; @@ -408,7 +502,9 @@ static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) { } return 0; } +#endif +#if 0 static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) { SMnode *pMnode = pMsg->pNode; SMqDoRebalanceMsg *pReq = pMsg->rpcMsg.pCont; @@ -422,7 +518,6 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) { if (pIter == NULL) break; SMqRebSubscribe *pRebSub = (SMqRebSubscribe *)pIter; SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pRebSub->key); - taosMemoryFreeClear(pRebSub->key); mInfo("mq rebalance subscription: %s, vgNum: %d, unassignedVg: %d", pSub->key, pSub->vgNum, (int32_t)taosArrayGetSize(pSub->unassignedVg)); @@ -562,6 +657,346 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) { mndTransDrop(pTrans); return 0; } +#endif + +static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) { + if (pInput->pTopic != NULL) { + // create subscribe + pOutput->pSub = mndCreateSub(pMnode, pInput->pTopic, pInput->pRebInfo->key); + } else { + pOutput->pSub = tCloneSubscribeObj(pInput->pOldSub); + } + int32_t totalVgNum = pOutput->pSub->vgNum; + + // 1. build temporary hash(vgId -> SMqRebOutputVg) to store modified vg + SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + + // 2. check and get actual removed consumers, put their vg into hash + int32_t removedNum = taosArrayGetSize(pInput->pRebInfo->removedConsumers); + int32_t actualRemoved = 0; + for (int32_t i = 0; i < removedNum; i++) { + int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->removedConsumers, i); + ASSERT(consumerId > 0); + SMqConsumerEpInSub *pEpInSub = taosHashGet(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t)); + ASSERT(pEpInSub); + if (pEpInSub) { + ASSERT(consumerId == pEpInSub->consumerId); + actualRemoved++; + int32_t consumerVgNum = taosArrayGetSize(pEpInSub->vgs); + for (int32_t j = 0; j < consumerVgNum; j++) { + SMqVgEp *pVgEp = taosArrayGetP(pEpInSub->vgs, j); + SMqRebOutputVg outputVg = { + .oldConsumerId = consumerId, + .newConsumerId = -1, + .pVgEp = pVgEp, + }; + taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); + } + taosHashRelease(pOutput->pSub->consumerHash, pEpInSub); + taosHashRemove(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t)); + // put into removed + taosArrayPush(pOutput->removedConsumers, &consumerId); + } + } + ASSERT(removedNum == actualRemoved); + + // if previously no consumer, there are vgs not assigned + { + int64_t key = -1; + SMqConsumerEpInSub *pEpInSub = taosHashGet(pOutput->pSub->consumerHash, &key, sizeof(int64_t)); + ASSERT(pEpInSub); + int32_t consumerVgNum = taosArrayGetSize(pEpInSub->vgs); + for (int32_t i = 0; i < consumerVgNum; i++) { + SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(pEpInSub->vgs); + SMqRebOutputVg rebOutput = { + .oldConsumerId = -1, + .newConsumerId = -1, + .pVgEp = pVgEp, + }; + taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &rebOutput, sizeof(SMqRebOutputVg)); + } + taosHashRelease(pOutput->pSub->consumerHash, pEpInSub); + } + + // 3. calc vg number of each consumer + int32_t actualConsumerNum = taosHashGetSize(pInput->pOldSub->consumerHash) - 1 + + taosArrayGetSize(pInput->pRebInfo->newConsumers) - + taosArrayGetSize(pInput->pRebInfo->removedConsumers); + int32_t afterRebConsumerNum = taosHashGetSize(pOutput->pSub->consumerHash) - 1; + ASSERT(afterRebConsumerNum == actualConsumerNum); + // calc num + int32_t minVgCnt = totalVgNum / actualConsumerNum; + int32_t imbConsumerNum = totalVgNum % actualConsumerNum; + + // 4. first scan: remove consumer more than wanted, put to remove hash + int32_t imbCnt = 0; + void *pIter = NULL; + while (1) { + pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); + if (pIter == NULL) break; + SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter; + ASSERT(pEpInSub->consumerId > 0); + int32_t consumerVgNum = taosArrayGetSize(pEpInSub->vgs); + // all old consumers still existing are touched + // TODO optimize: touch only consumer whose vgs changed + taosArrayPush(pOutput->touchedConsumers, &pEpInSub->consumerId); + if (consumerVgNum > minVgCnt) { + if (imbCnt < imbConsumerNum) { + if (consumerVgNum == minVgCnt + 1) { + continue; + } else { + // pop until equal minVg + 1 + while (taosArrayGetSize(pEpInSub->vgs) > minVgCnt + 1) { + SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(pEpInSub->vgs); + SMqRebOutputVg outputVg = { + .oldConsumerId = pEpInSub->consumerId, + .newConsumerId = -1, + .pVgEp = pVgEp, + }; + taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); + } + imbCnt++; + } + } else { + // pop until equal minVg + while (taosArrayGetSize(pEpInSub->vgs) > minVgCnt) { + SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(pEpInSub->vgs); + SMqRebOutputVg outputVg = { + .oldConsumerId = pEpInSub->consumerId, + .newConsumerId = -1, + .pVgEp = pVgEp, + }; + taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); + } + } + } + } + + // 5. add new consumer into sub + { + int32_t consumerNum = taosArrayGetSize(pInput->pRebInfo->newConsumers); + for (int32_t i = 0; i < consumerNum; i++) { + int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->newConsumers, i); + ASSERT(consumerId > 0); + SMqConsumerEpInSub newConsumerEp; + newConsumerEp.consumerId = consumerId; + newConsumerEp.vgs = taosArrayInit(0, sizeof(void *)); + taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, + sizeof(SMqConsumerEpInSub)); + taosArrayPush(pOutput->newConsumers, &consumerId); + } + } + + // 6. second scan: find consumer do not have enough vg, extract from temporary hash and assign to new consumer. + // All related vg should be put into rebVgs + SMqRebOutputVg *pRebVg = NULL; + void *pRemovedIter = NULL; + pIter = NULL; + while (1) { + pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); + if (pIter == NULL) break; + SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter; + ASSERT(pEpInSub->consumerId > 0); + /*int32_t consumerVgNum = taosArrayGetSize(pEpInSub->vgs);*/ + if (imbCnt < imbConsumerNum) { + imbCnt++; + // push until equal minVg + 1 + while (taosArrayGetSize(pEpInSub->vgs) < minVgCnt + 1) { + // iter hash and find one vg + pRemovedIter = taosHashIterate(pHash, pRemovedIter); + ASSERT(pRemovedIter); + pRebVg = (SMqRebOutputVg *)pRemovedIter; + // push + taosArrayPush(pEpInSub->vgs, &pRebVg->pVgEp); + pRebVg->newConsumerId = pEpInSub->consumerId; + taosArrayPush(pOutput->rebVgs, pRebVg); + } + } else { + // push until equal minVg + while (taosArrayGetSize(pEpInSub->vgs) < minVgCnt) { + // iter hash and find one vg + pRemovedIter = taosHashIterate(pHash, pRemovedIter); + ASSERT(pRemovedIter); + pRebVg = (SMqRebOutputVg *)pRemovedIter; + // push + taosArrayPush(pEpInSub->vgs, &pRebVg->pVgEp); + pRebVg->newConsumerId = pEpInSub->consumerId; + taosArrayPush(pOutput->rebVgs, pRebVg); + } + } + } + + // 7. handle unassigned vg + if (taosHashGetSize(pOutput->pSub->consumerHash) != 1) { + // if has consumer, vg should be all assigned + pRemovedIter = taosHashIterate(pHash, pRemovedIter); + ASSERT(pRemovedIter == NULL); + } else { + // if all consumer is removed, put all vg into unassigned + int64_t key = -1; + SMqConsumerEpInSub *pEpInSub = taosHashGet(pOutput->pSub->consumerHash, &key, sizeof(int64_t)); + ASSERT(pEpInSub); + ASSERT(pEpInSub->consumerId == -1); + + pIter = NULL; + SMqRebOutputVg *pRebOutput = NULL; + while (1) { + pIter = taosHashIterate(pHash, pIter); + if (pIter == NULL) break; + pRebOutput = (SMqRebOutputVg *)pIter; + ASSERT(pRebOutput->newConsumerId == -1); + taosArrayPush(pEpInSub->vgs, pRebOutput->pVgEp); + } + } + + // 8. generate logs + + // 9. clear + taosHashCleanup(pHash); + + return 0; +} + +static int32_t mndPersistRebResult(SMnode *pMnode, SNodeMsg *pMsg, const SMqRebOutputObj *pOutput) { + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_REBALANCE, &pMsg->rpcMsg); + if (pTrans == NULL) { + return -1; + } + // make txn: + // 1. redo action: action to all vg + const SArray *rebVgs = pOutput->rebVgs; + int32_t vgNum = taosArrayGetSize(rebVgs); + for (int32_t i = 0; i < vgNum; i++) { + SMqRebOutputVg *pRebVg = taosArrayGet(rebVgs, i); + if (mndPersistSubChangeVgReq(pMnode, pTrans, pOutput->pSub->key, pRebVg) < 0) { + goto REB_FAIL; + } + } + + // 2. redo log: subscribe and vg assignment + // subscribe + if (mndSetSubRedoLogs(pMnode, pTrans, pOutput->pSub) != 0) { + goto REB_FAIL; + } + + // 3. commit log: consumer to update status and epoch + // 3.1 set touched consumer + int32_t consumerNum = taosArrayGetSize(pOutput->touchedConsumers); + for (int32_t i = 0; i < consumerNum; i++) { + int64_t consumerId = *(int64_t *)taosArrayGet(pOutput->touchedConsumers, i); + SMqConsumerObj *pConsumerOld = mndAcquireConsumer(pMnode, consumerId); + SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumerOld->consumerId, pConsumerOld->cgroup); + pConsumerNew->updateType = CONSUMER_UPDATE__TOUCH; + mndReleaseConsumer(pMnode, pConsumerOld); + if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) { + goto REB_FAIL; + } + } + // 3.2 set new consumer + consumerNum = taosArrayGetSize(pOutput->newConsumers); + for (int32_t i = 0; i < consumerNum; i++) { + int64_t consumerId = *(int64_t *)taosArrayGet(pOutput->removedConsumers, i); + ASSERT(consumerId > 0); + SMqConsumerObj *pConsumerOld = mndAcquireConsumer(pMnode, consumerId); + SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumerOld->consumerId, pConsumerOld->cgroup); + pConsumerNew->updateType = CONSUMER_UPDATE__ADD; + char *topic = taosMemoryCalloc(1, TSDB_TOPIC_FNAME_LEN); + char cgroup[TSDB_CGROUP_LEN]; + mndSplitSubscribeKey(pOutput->pSub->key, topic, cgroup); + taosArrayPush(pConsumerNew->rebNewTopics, &topic); + mndReleaseConsumer(pMnode, pConsumerOld); + if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) { + goto REB_FAIL; + } + } + + // 3.3 set removed consumer + consumerNum = taosArrayGetSize(pOutput->removedConsumers); + for (int32_t i = 0; i < consumerNum; i++) { + int64_t consumerId = *(int64_t *)taosArrayGet(pOutput->removedConsumers, i); + ASSERT(consumerId > 0); + SMqConsumerObj *pConsumerOld = mndAcquireConsumer(pMnode, consumerId); + SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumerOld->consumerId, pConsumerOld->cgroup); + pConsumerNew->updateType = CONSUMER_UPDATE__REMOVE; + char *topic = taosMemoryCalloc(1, TSDB_TOPIC_FNAME_LEN); + char cgroup[TSDB_CGROUP_LEN]; + mndSplitSubscribeKey(pOutput->pSub->key, topic, cgroup); + taosArrayPush(pConsumerNew->rebRemovedTopics, &topic); + mndReleaseConsumer(pMnode, pConsumerOld); + if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) { + goto REB_FAIL; + } + } + // 4. commit log: modification log + if (mndTransPrepare(pMnode, pTrans) != 0) goto REB_FAIL; + + mndTransDrop(pTrans); + return 0; + +REB_FAIL: + mndTransDrop(pTrans); + return -1; +} + +static int32_t mndProcessRebalanceReq(SNodeMsg *pMsg) { + SMnode *pMnode = pMsg->pNode; + SMqDoRebalanceMsg *pReq = pMsg->rpcMsg.pCont; + void *pIter = NULL; + + mInfo("mq rebalance start"); + + while (1) { + pIter = taosHashIterate(pReq->rebSubHash, pIter); + if (pIter == NULL) break; + SMqRebInputObj rebInput = {0}; + SMqRebOutputObj rebOutput = {0}; + SMqRebSubscribe *pRebSub = (SMqRebSubscribe *)pIter; + SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pRebSub->key); + + if (pSub == NULL) { + taosRLockLatch(&pSub->lock); + // split sub key and extract topic + char topic[TSDB_TOPIC_FNAME_LEN]; + char cgroup[TSDB_CGROUP_LEN]; + mndSplitSubscribeKey(pSub->key, topic, cgroup); + SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); + ASSERT(pTopic); + taosRLockLatch(&pTopic->lock); + rebInput.pTopic = pTopic; + } + + rebInput.pRebInfo = pRebSub; + rebInput.pOldSub = pSub; + + int32_t unassignedVgNum = 0; + int64_t key = -1; + SMqConsumerEpInSub *pEpInSub = taosHashGet(pSub->consumerHash, &key, sizeof(int64_t)); + if (pEpInSub != NULL) { + ASSERT(pEpInSub->consumerId == key); + unassignedVgNum = taosArrayGetSize(pEpInSub->vgs); + } + mInfo("mq rebalance subscription: %s, vgNum: %d, unassignedVg: %d", pSub->key, pSub->vgNum, unassignedVgNum); + + // TODO replace assert with error check + ASSERT(mndDoRebalance(pMnode, &rebInput, &rebOutput) == 0); + ASSERT(mndPersistRebResult(pMnode, pMsg, &rebOutput) == 0); + + if (rebInput.pTopic) { + SMqTopicObj *pTopic = (SMqTopicObj *)rebInput.pTopic; + taosRUnLockLatch(&pTopic->lock); + mndReleaseTopic(pMnode, pTopic); + taosRUnLockLatch(&pSub->lock); + mndReleaseSubscribe(pMnode, pSub); + } + } + + // reset flag + atomic_store_8(pReq->mqInReb, 0); + mInfo("mq rebalance completed successfully"); + taosHashCleanup(pReq->rebSubHash); + + return 0; +} static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup, const SMqConsumerEp *pConsumerEp) { @@ -697,16 +1132,23 @@ static int32_t mndSubActionInsert(SSdb *pSdb, SMqSubscribeObj *pSub) { static int32_t mndSubActionDelete(SSdb *pSdb, SMqSubscribeObj *pSub) { mTrace("subscribe:%s, perform delete action", pSub->key); - tDeleteSMqSubscribeObj(pSub); + tDeleteSubscribeObj(pSub); return 0; } static int32_t mndSubActionUpdate(SSdb *pSdb, SMqSubscribeObj *pOldSub, SMqSubscribeObj *pNewSub) { mTrace("subscribe:%s, perform update action", pOldSub->key); + taosWLockLatch(&pOldSub->lock); + + SHashObj *tmp = pOldSub->consumerHash; + pOldSub->consumerHash = pNewSub->consumerHash; + pNewSub->consumerHash = tmp; + + taosWUnLockLatch(&pOldSub->lock); return 0; } -static int32_t mndMakeSubscribeKey(char *key, const char *cgroup, const char *topicName) { +int32_t mndMakeSubscribeKey(char *key, const char *cgroup, const char *topicName) { int32_t tlen = strlen(cgroup); memcpy(key, cgroup, tlen); key[tlen] = TMQ_SEPARATOR; @@ -739,6 +1181,7 @@ void mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub) { sdbRelease(pSdb, pSub); } +#if 0 static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) { SMnode *pMnode = pMsg->pNode; char *msgStr = pMsg->rpcMsg.pCont; @@ -901,6 +1344,7 @@ static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) { if (!createConsumer) mndReleaseConsumer(pMnode, pConsumer); return TSDB_CODE_MND_ACTION_IN_PROGRESS; } +#endif static int32_t mndProcessSubscribeInternalRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index a74b26a386..84f52cc2fc 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "tarray.h" +#include "tcoding.h" SArray* taosArrayInit(size_t size, size_t elemSize) { assert(elemSize > 0); @@ -312,7 +313,14 @@ void* taosArrayDestroy(SArray* pArray) { return NULL; } -void taosArrayDestroyEx(SArray* pArray, void (*fp)(void*)) { +void taosArrayDestroyP(SArray* pArray, FDelete fp) { + for (int32_t i = 0; i < pArray->size; i++) { + fp(*(void**)TARRAY_GET_ELEM(pArray, i)); + } + taosArrayDestroy(pArray); +} + +void taosArrayDestroyEx(SArray* pArray, FDelete fp) { if (pArray == NULL) { return; } @@ -421,6 +429,37 @@ static void taosArrayInsertSort(SArray* pArray, __ext_compar_fn_t fn, const void return; } +SArray* taosArrayDeepCopy(const SArray* pSrc, FCopy deepCopy) { + SArray* pArray = taosArrayInit(pSrc->size, pSrc->elemSize); + for (int32_t i = 0; i < pSrc->size; i++) { + void* clone = deepCopy(taosArrayGetP(pSrc, i)); + taosArrayPush(pArray, &clone); + } + return pArray; +} + +int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode) { + int32_t tlen = 0; + int32_t sz = pArray->size; + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + void* data = taosArrayGetP(pArray, i); + tlen += encode(buf, data); + } + return tlen; +} + +void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz) { + int32_t sz; + buf = taosDecodeFixedI32(buf, &sz); + *pArray = taosArrayInit(sz, sizeof(void*)); + for (int32_t i = 0; i < sz; i++) { + void* data = taosMemoryCalloc(1, dataSz); + buf = decode(buf, data); + } + return (void*)buf; +} + // order array void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param) { taosArrayGetSize(pArray) > 8 ? taosArrayQuickSort(pArray, fn, param) : taosArrayInsertSort(pArray, fn, param); From 19bbd6806072955df859c4015496449a72293dd8 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 19 Apr 2022 17:11:53 +0800 Subject: [PATCH 20/47] remove commit --- tests/test/c/tmqSim.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index eaca8f151e..5ea15a6622 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -354,6 +354,7 @@ void loop_consume(tmq_t* tmq) { if (0 != g_stConfInfo.showMsgFlag) { /*msg_process(tmqMsg);*/ } + /*tmq_commit(tmq, NULL, false);*/ tmq_message_destroy(tmqMsg); } else { break; From 412e2cbb0d3631017c81783fdfb8d18ca67f304f Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 19 Apr 2022 17:52:04 +0800 Subject: [PATCH 21/47] refactor: add test casese for json compare logic --- include/util/tdef.h | 5 - source/libs/parser/src/parUtil.c | 4 +- source/libs/scalar/src/scalar.c | 88 +++++++++------ source/libs/scalar/src/sclvector.c | 23 +++- .../libs/scalar/test/scalar/scalarTests.cpp | 103 +++++++++++++++++- 5 files changed, 177 insertions(+), 46 deletions(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index eb7a51cd7e..4c3126caa9 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -272,11 +272,6 @@ typedef enum ELogicConditionType { #define TSDB_MAX_TAG_CONDITIONS 1024 #define TSDB_MAX_JSON_TAG_LEN 16384 -#define TSDB_JSON_PLACEHOLDER 0x7F -#define TSDB_JSON_null 0x00 -#define TSDB_JSON_NOT_NULL 0x01 -#define TSDB_JSON_NULL 0x00 - #define TSDB_AUTH_LEN 16 #define TSDB_PASSWORD_LEN 32 diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index f3eb0c5ebc..0ff5843736 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -276,7 +276,7 @@ static bool isValidateTag(char *input) { int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* pMsgBuf, int16_t startColId){ // set json NULL data - uint8_t jsonNULL = TSDB_JSON_NULL; + uint8_t jsonNULL = TSDB_DATA_TYPE_NULL; int jsonIndex = startColId + 1; if (!json || strcasecmp(json, TSDB_DATA_NULL_STR_L) == 0){ tdAddColToKVRow(kvRowBuilder, jsonIndex, &jsonNULL, CHAR_BYTES); @@ -328,7 +328,7 @@ int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* p strncpy(varDataVal(tagKV), jsonKey, keyLen); varDataSetLen(tagKV, keyLen); if(taosHashGetSize(keyHash) == 0){ - uint8_t jsonNotNULL = TSDB_JSON_NOT_NULL; + uint8_t jsonNotNULL = TSDB_DATA_TYPE_JSON; tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNotNULL, CHAR_BYTES); // add json type } taosHashPut(keyHash, jsonKey, keyLen, &keyLen, CHAR_BYTES); // add key to hash to remove dumplicate, value is useless diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 9c4d2d94dd..2fd688a233 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -604,39 +604,55 @@ EDealRes sclWalkOperator(SNode* pNode, SScalarCtx *ctx) { return DEAL_RES_CONTINUE; } -//EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) { -// STargetNode *target = (STargetNode *)pNode; -// -// if (target->dataBlockId >= taosArrayGetSize(ctx->pBlockList)) { -// sclError("target tupleId is too big, tupleId:%d, dataBlockNum:%d", target->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList)); -// ctx->code = TSDB_CODE_QRY_INVALID_INPUT; -// return DEAL_RES_ERROR; -// } -// -// SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, target->dataBlockId); -// if (target->slotId >= taosArrayGetSize(block->pDataBlock)) { -// sclError("target slot not exist, dataBlockId:%d, slotId:%d, dataBlockNum:%d", target->dataBlockId, target->slotId, (int32_t)taosArrayGetSize(block->pDataBlock)); -// ctx->code = TSDB_CODE_QRY_INVALID_INPUT; -// return DEAL_RES_ERROR; -// } -// -// // if block->pDataBlock is not enough, there are problems if target->slotId bigger than the size of block->pDataBlock, -// SColumnInfoData *col = taosArrayGet(block->pDataBlock, target->slotId); -// -// SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES); -// if (NULL == res) { -// sclError("no valid res in hash, node:%p, type:%d", target->pExpr, nodeType(target->pExpr)); -// ctx->code = TSDB_CODE_QRY_APP_ERROR; -// return DEAL_RES_ERROR; -// } -// -// colDataAssign(col, res->columnData, res->numOfRows); -// block->info.rows = res->numOfRows; -// -// sclFreeParam(res); -// taosHashRemove(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES); -// return DEAL_RES_CONTINUE; -//} +EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) { + STargetNode *target = (STargetNode *)pNode; + + if (target->dataBlockId >= taosArrayGetSize(ctx->pBlockList)) { + sclError("target tupleId is too big, tupleId:%d, dataBlockNum:%d", target->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList)); + ctx->code = TSDB_CODE_QRY_INVALID_INPUT; + return DEAL_RES_ERROR; + } + + int32_t index = -1; + for(int32_t i = 0; i < taosArrayGetSize(ctx->pBlockList); ++i) { + SSDataBlock* pb = taosArrayGetP(ctx->pBlockList, i); + if (pb->info.blockId == target->dataBlockId) { + index = i; + break; + } + } + + if (index == -1) { + sclError("column tupleId is too big, tupleId:%d, dataBlockNum:%d", target->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList)); + ctx->code = TSDB_CODE_QRY_INVALID_INPUT; + return DEAL_RES_ERROR; + } + + SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, index); + + if (target->slotId >= taosArrayGetSize(block->pDataBlock)) { + sclError("target slot not exist, dataBlockId:%d, slotId:%d, dataBlockNum:%d", target->dataBlockId, target->slotId, (int32_t)taosArrayGetSize(block->pDataBlock)); + ctx->code = TSDB_CODE_QRY_INVALID_INPUT; + return DEAL_RES_ERROR; + } + + // if block->pDataBlock is not enough, there are problems if target->slotId bigger than the size of block->pDataBlock, + SColumnInfoData *col = taosArrayGet(block->pDataBlock, target->slotId); + + SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES); + if (NULL == res) { + sclError("no valid res in hash, node:%p, type:%d", target->pExpr, nodeType(target->pExpr)); + ctx->code = TSDB_CODE_QRY_APP_ERROR; + return DEAL_RES_ERROR; + } + + colDataAssign(col, res->columnData, res->numOfRows); + block->info.rows = res->numOfRows; + + sclFreeParam(res); + taosHashRemove(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES); + return DEAL_RES_CONTINUE; +} EDealRes sclCalcWalker(SNode* pNode, void* pContext) { if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)) { @@ -656,9 +672,9 @@ EDealRes sclCalcWalker(SNode* pNode, void* pContext) { return sclWalkOperator(pNode, ctx); } -// if (QUERY_NODE_TARGET == nodeType(pNode)) { -// return sclWalkTarget(pNode, ctx); -// } + if (QUERY_NODE_TARGET == nodeType(pNode)) { + return sclWalkTarget(pNode, ctx); + } sclError("invalid node type for scalar calculating, type:%d", nodeType(pNode)); ctx->code = TSDB_CODE_QRY_INVALID_INPUT; diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index e97831b1f6..7ad200befc 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -767,6 +767,17 @@ static void doReleaseVec(SColumnInfoData* pCol, int32_t type) { } char *getJsonValue(char *json, char *key){ //todo + int16_t cols = kvRowNCols(json); + for (int i = 0; i < cols; ++i) { + SColIdx *pColIdx = kvRowColIdxAt(json, i); + char *data = kvRowColVal(json, pColIdx); + if(i == 0 && *data == TSDB_DATA_TYPE_NULL){ + return NULL; + } + if(strncmp(key, varDataVal(data), varDataLen(data)) == 0){ + return data + varDataTLen(data); + } + } return NULL; } @@ -844,7 +855,11 @@ void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) + getVectorDoubleValueFnRight(pRightCol->pData, i); + if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore + } + *output = getVectorDoubleValueFnLeft(LEFT_COL, i) + getVectorDoubleValueFnRight(RIGHT_COL, i); } pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull); @@ -957,7 +972,11 @@ void vectorMathSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) - getVectorDoubleValueFnRight(pRightCol->pData, i); + if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore + } + *output = getVectorDoubleValueFnLeft(LEFT_COL, i) - getVectorDoubleValueFnRight(RIGHT_COL, i); } pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull); diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 61ef2fdce2..975effc63a 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -92,6 +92,7 @@ void scltAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *s blockDataEnsureCapacity(res, rows); *dataBlockId = taosArrayGetSize(pBlockList) - 1; + res->info.blockId = *dataBlockId; *slotId = 0; } else { SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(pBlockList); @@ -124,6 +125,56 @@ void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) { *pNode = (SNode *)vnode; } +void scltMakeJsonColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { + SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_COLUMN); + SColumnNode *rnode = (SColumnNode *)node; + rnode->node.resType.type = dataType; + rnode->node.resType.bytes = dataBytes; + rnode->dataBlockId = 0; + + if (NULL == *block) { + SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); + res->info.numOfCols = 3; + res->info.rows = rowNum; + res->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData)); + for (int32_t i = 0; i < 2; ++i) { + SColumnInfoData idata = {{0}}; + idata.info.type = TSDB_DATA_TYPE_NULL; + idata.info.bytes = 10; + idata.info.colId = i + 1; + + int32_t size = idata.info.bytes * rowNum; + idata.pData = (char *)taosMemoryCalloc(1, size); + taosArrayPush(res->pDataBlock, &idata); + } + + SColumnInfoData idata = {{0}}; + idata.info.type = dataType; + idata.info.bytes = dataBytes; + idata.info.colId = 3; + int32_t size = idata.info.bytes * rowNum; + idata.pData = (char *)taosMemoryCalloc(1, size); + taosArrayPush(res->pDataBlock, &idata); + + blockDataEnsureCapacity(res, rowNum); + + SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + for (int32_t i = 0; i < rowNum; ++i) { + uint32_t len = pColumnInfoData->varmeta.length; + pColumnInfoData->varmeta.offset[i] = len; + + memcpy(pColumnInfoData->pData + len, value, varDataLen(value)); + pColumnInfoData->varmeta.length += varDataLen(value); + } + + rnode->slotId = 2; + rnode->colId = 3; + + *block = res; + } + + *pNode = (SNode *)rnode; +} void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_COLUMN); @@ -909,6 +960,56 @@ TEST(constantTest, greater_and_lower) { nodesDestroyNode(res); } +TEST(columnTest, smallint_value_add_json_column) { + scltInitLogFile(); + + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int32_t leftv = 1; + char *rightv= "{"k1":4,"k2":"hello","k3":null,"k4":true,"k5":5.44}"; + char key[10] = {0}; + memcpy(varDataVal(key), "k1", 2); + varDataLen(key) = 2; + + SKVRowBuilder kvRowBuilder; + tdInitKVRowBuilder(&kvRowBuilder); + parseJsontoTagData(rightv, &kvRowBuilder, NULL, 0); + SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); + + double eRes[5] = {1.0, -4, -3, 24, 101}; + + SSDataBlock *src = NULL; + int32_t rowNum = 1; + + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, key); + scltMakeJsonColumnNode(&pLeft, &src, TSDB_DATA_TYPE_JSON, varDataLen(row), rowNum, row); + scltMakeOpNode(&opNode, OP_TYPE_JSON_GET_VALUE, TSDB_DATA_TYPE_JSON, pLeft, pRight); + + SNode *right = NULL; + scltMakeValueNode(&right, TSDB_DATA_TYPE_INT, &leftv); + scltMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, opNode, right); + + SArray *blockList = taosArrayInit(2, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = createColumnInfo(1, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); + + int32_t code = scalarCalculate(opNode, blockList, NULL); + ASSERT_EQ(code, 0); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_DOUBLE); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)colDataGetData(column, i)), eRes[i]); + } + + taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); +} + TEST(columnTest, smallint_value_add_int_column) { scltInitLogFile(); @@ -928,7 +1029,7 @@ TEST(columnTest, smallint_value_add_int_column) { int16_t dataBlockId = 0, slotId = 0; scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo); scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); - + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); From 0f13cee876b6dcd3a10d1f730cac0f69bf60d32f Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 19 Apr 2022 20:16:17 +0800 Subject: [PATCH 22/47] stmt --- include/client/taos.h | 2 +- source/libs/parser/inc/parInsertData.h | 4 +- source/libs/parser/src/parInsert.c | 6 + tests/script/api/batchprepare.c | 814 ++++++++++++++++++++----- tests/script/api/makefile | 4 +- 5 files changed, 674 insertions(+), 156 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index a0cf8e8296..ed00fbe69a 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -95,7 +95,7 @@ typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code); typedef struct TAOS_BIND_v2 { int buffer_type; void *buffer; - uintptr_t buffer_length; + int32_t buffer_length; int32_t *length; char *is_null; int num; diff --git a/source/libs/parser/inc/parInsertData.h b/source/libs/parser/inc/parInsertData.h index 6b45d16d1a..131fed7413 100644 --- a/source/libs/parser/inc/parInsertData.h +++ b/source/libs/parser/inc/parInsertData.h @@ -98,7 +98,7 @@ static FORCE_INLINE void getSTSRowAppendInfo(uint8_t rowType, SParsedDataColInfo *toffset = (spd->cols + schemaIdx)->toffset; // the offset of firstPart *colIdx = schemaIdx; } else { - *toffset = idx * sizeof(SColIdx); // the offset of SColIdx + *toffset = idx * sizeof(SKvRowIdx); // the offset of SColIdx *colIdx = idx; } } else { @@ -108,7 +108,7 @@ static FORCE_INLINE void getSTSRowAppendInfo(uint8_t rowType, SParsedDataColInfo *toffset = (spd->cols + schemaIdx)->toffset; *colIdx = schemaIdx; } else { - *toffset = ((spd->colIdxInfo + idx)->finalIdx) * sizeof(SColIdx); + *toffset = ((spd->colIdxInfo + idx)->finalIdx) * sizeof(SKvRowIdx); *colIdx = (spd->colIdxInfo + idx)->finalIdx; } } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 08043178d2..0cc2b30fd0 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1319,6 +1319,8 @@ int32_t qBindStmtColsValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32 SMemParam param = {.rb = pBuilder}; SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; + CHECK_CODE(initRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo)); + CHECK_CODE(allocateMemForSize(pDataBlock, extendedRowSize * bind->num)); for (int32_t r = 0; r < bind->num; ++r) { @@ -1333,6 +1335,10 @@ int32_t qBindStmtColsValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32 getSTSRowAppendInfo(pBuilder->rowType, spd, c, ¶m.toffset, ¶m.colIdx); if (bind[c].is_null && bind[c].is_null[r]) { + if (pColSchema->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + return buildInvalidOperationMsg(&pBuf, "primary timestamp should not be NULL"); + } + CHECK_CODE(MemRowAppend(&pBuf, NULL, 0, ¶m)); } else { int32_t colLen = pColSchema->bytes; diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index da291c049e..0740560f32 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -10,16 +10,68 @@ #include "../../../include/client/taos.h" typedef struct { - TAOS *taos; - int idx; -}T_par; + int64_t* tsData; + bool* boolData; + int8_t* tinyData; + uint8_t* utinyData; + int16_t* smallData; + uint16_t* usmallData; + int32_t* intData; + uint32_t* uintData; + int64_t* bigData; + uint64_t* ubigData; + float* floatData; + double* doubleData; + char* binaryData; + char* isNull; + int32_t* binaryLen; + TAOS_BIND_v2* pBind; + char* sql; + int32_t* colTypes; + int32_t colNum; +} BindData; + +int32_t gVarCharSize = 10; +int32_t gVarCharLen = 5; + +int insertAllCols(TAOS_STMT *stmt); +int insertSpecifyCols(TAOS_STMT *stmt); + +int32_t shortColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_INT}; +int32_t longColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_BOOL, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_UTINYINT, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_USMALLINT, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_UINT, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_UBIGINT, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_BINARY, TSDB_DATA_TYPE_NCHAR}; + +#define tListLen(x) (sizeof(x) / sizeof((x)[0])) + + +typedef struct { + char caseDesc[128]; + int32_t colNum; + int32_t *colList; + bool autoCreate; + bool fullCol; + int32_t (*runFn)(TAOS_STMT*); + int32_t tblNum; + int32_t rowNum; + int32_t bindRowNum; + int32_t bindColNum; + int32_t bindNullNum; + int32_t runTimes; +} CaseCfg; + +CaseCfg gCase[] = { +// {"insert:all columns", tListLen(shortColList), shortColList, false, true, insertAllCols, 1, 10, 10, 2, 0, 10}, +// {"insert:all columns", tListLen(shortColList), shortColList, false, true, insertAllCols, 10, 100, 10, 2, 0, 10}, + {"insert:all columns", tListLen(longColList), longColList, false, true, insertAllCols, 10, 10, 2, 14, 0, 1}, +// {"insert:all columns", tListLen(longColList), longColList, false, false, insertSpecifyCols, 10, 10, 2, 6, 0, 1}, +}; + +CaseCfg *gCurCase = NULL; -void taosMsleep(int mseconds); int32_t taosGetTimeOfDay(struct timeval *tv) { return gettimeofday(tv, NULL); } -void *taosMemoryMalloc(int32_t size) { +void *taosMemoryMalloc(uint64_t size) { return malloc(size); } @@ -32,50 +84,396 @@ void taosMemoryFree(const void *ptr) { return free((void*)ptr); } +static int64_t taosGetTimestampMs() { + struct timeval systemTime; + taosGetTimeOfDay(&systemTime); + return (int64_t)systemTime.tv_sec * 1000L + (int64_t)systemTime.tv_usec/1000; +} + static int64_t taosGetTimestampUs() { struct timeval systemTime; taosGetTimeOfDay(&systemTime); return (int64_t)systemTime.tv_sec * 1000000L + (int64_t)systemTime.tv_usec; } +bool colExists(TAOS_BIND_v2* pBind, int32_t dataType) { + int32_t i = 0; + while (true) { + if (0 == pBind[i].buffer_type) { + return false; + } -int stmt_allcol_func1(TAOS_STMT *stmt) { - struct { - int64_t ts; - int32_t v4; - } v = {0}; - int32_t len[10] = {sizeof(v.ts), sizeof(v.v4)}; + if (pBind[i].buffer_type == dataType) { + return true; + } + + ++i; + } +} + +void generateSql(BindData *data) { + int32_t len = sprintf(data->sql, "insert into %s ", (gCurCase->tblNum > 1 ? "? " : "m0 ")); + if (!gCurCase->fullCol) { + len += sprintf(data->sql + len, "("); + for (int c = 0; c < gCurCase->bindColNum; ++c) { + if (c) { + len += sprintf(data->sql + len, ","); + } + switch (data->pBind[c].buffer_type) { + case TSDB_DATA_TYPE_BOOL: + len += sprintf(data->sql + len, "booldata"); + break; + case TSDB_DATA_TYPE_TINYINT: + len += sprintf(data->sql + len, "tinydata"); + break; + case TSDB_DATA_TYPE_SMALLINT: + len += sprintf(data->sql + len, "smalldata"); + break; + case TSDB_DATA_TYPE_INT: + len += sprintf(data->sql + len, "intdata"); + break; + case TSDB_DATA_TYPE_BIGINT: + len += sprintf(data->sql + len, "bigdata"); + break; + case TSDB_DATA_TYPE_FLOAT: + len += sprintf(data->sql + len, "floatdata"); + break; + case TSDB_DATA_TYPE_DOUBLE: + len += sprintf(data->sql + len, "doubledata"); + break; + case TSDB_DATA_TYPE_VARCHAR: + len += sprintf(data->sql + len, "binarydata"); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + len += sprintf(data->sql + len, "ts"); + break; + case TSDB_DATA_TYPE_NCHAR: + len += sprintf(data->sql + len, "nchardata"); + break; + case TSDB_DATA_TYPE_UTINYINT: + len += sprintf(data->sql + len, "utinydata"); + break; + case TSDB_DATA_TYPE_USMALLINT: + len += sprintf(data->sql + len, "usmalldata"); + break; + case TSDB_DATA_TYPE_UINT: + len += sprintf(data->sql + len, "uintdata"); + break; + case TSDB_DATA_TYPE_UBIGINT: + len += sprintf(data->sql + len, "ubigdata"); + break; + default: + printf("invalid col type:%d", data->pBind[c].buffer_type); + exit(1); + } + } + + len += sprintf(data->sql + len, ") "); + } + + len += sprintf(data->sql + len, "values ("); + for (int c = 0; c < gCurCase->bindColNum; ++c) { + if (c) { + len += sprintf(data->sql + len, ","); + } + len += sprintf(data->sql + len, "?"); + } + len += sprintf(data->sql + len, ")"); - TAOS_BIND_v2 params[10]; - params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[0].buffer_length = sizeof(v.ts); - params[0].buffer = &v.ts; - params[0].length = &len[0]; - params[0].is_null = NULL; - params[0].num = 1; +} - params[1].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[1].buffer_length = sizeof(v.v4); - params[1].buffer = &v.v4; - params[1].length = &len[1]; - params[1].is_null = NULL; - params[1].num = 1; +void generateDataType(BindData *data, int32_t bindIdx, int32_t colIdx, int32_t *dataType) { + if (bindIdx < gCurCase->bindColNum) { + if (colIdx) { + if (!gCurCase->multiCol) { + *dataType = TSDB_DATA_TYPE_INT; + return; + } + + while (true) { + *dataType = rand() % (TSDB_DATA_TYPE_MAX - 1) + 1; + if (*dataType == TSDB_DATA_TYPE_JSON || *dataType == TSDB_DATA_TYPE_DECIMAL + || *dataType == TSDB_DATA_TYPE_BLOB || *dataType == TSDB_DATA_TYPE_MEDIUMBLOB + || *dataType == TSDB_DATA_TYPE_VARBINARY) { + continue; + } - char *sql = "insert into m0 values(?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + if (colExists(data->pBind, *dataType)) { + continue; + } + + break; + } + } + } else { + *dataType = data->pBind[bindIdx%gCurCase->bindColNum].buffer_type; + } +} + +int32_t prepareColData(BindData *data, int32_t bindIdx, int32_t rowIdx, int32_t colIdx) { + int32_t dataType = TSDB_DATA_TYPE_TIMESTAMP; + + generateDataType(data, bindIdx, colIdx, &dataType); + + switch (dataType) { + case TSDB_DATA_TYPE_BOOL: + data->pBind[bindIdx].buffer_length = sizeof(bool); + data->pBind[bindIdx].buffer = data->boolData + rowIdx; + data->pBind[bindIdx].length = NULL; + data->pBind[bindIdx].is_null = data->isNull; + break; + case TSDB_DATA_TYPE_TINYINT: + data->pBind[bindIdx].buffer_length = sizeof(int8_t); + data->pBind[bindIdx].buffer = data->tinyData + rowIdx; + data->pBind[bindIdx].length = NULL; + data->pBind[bindIdx].is_null = data->isNull; + break; + case TSDB_DATA_TYPE_SMALLINT: + data->pBind[bindIdx].buffer_length = sizeof(int16_t); + data->pBind[bindIdx].buffer = data->smallData + rowIdx; + data->pBind[bindIdx].length = NULL; + data->pBind[bindIdx].is_null = data->isNull; + break; + case TSDB_DATA_TYPE_INT: + data->pBind[bindIdx].buffer_length = sizeof(int32_t); + data->pBind[bindIdx].buffer = data->intData + rowIdx; + data->pBind[bindIdx].length = NULL; + data->pBind[bindIdx].is_null = data->isNull; + break; + case TSDB_DATA_TYPE_BIGINT: + data->pBind[bindIdx].buffer_length = sizeof(int64_t); + data->pBind[bindIdx].buffer = data->bigData + rowIdx; + data->pBind[bindIdx].length = NULL; + data->pBind[bindIdx].is_null = data->isNull; + break; + case TSDB_DATA_TYPE_FLOAT: + data->pBind[bindIdx].buffer_length = sizeof(float); + data->pBind[bindIdx].buffer = data->floatData + rowIdx; + data->pBind[bindIdx].length = NULL; + data->pBind[bindIdx].is_null = data->isNull; + break; + case TSDB_DATA_TYPE_DOUBLE: + data->pBind[bindIdx].buffer_length = sizeof(double); + data->pBind[bindIdx].buffer = data->doubleData + rowIdx; + data->pBind[bindIdx].length = NULL; + data->pBind[bindIdx].is_null = data->isNull; + break; + case TSDB_DATA_TYPE_VARCHAR: + data->pBind[bindIdx].buffer_length = gVarCharSize; + data->pBind[bindIdx].buffer = data->binaryData + rowIdx; + data->pBind[bindIdx].length = data->binaryLen; + data->pBind[bindIdx].is_null = data->isNull; + break; + case TSDB_DATA_TYPE_TIMESTAMP: + data->pBind[bindIdx].buffer_length = sizeof(int64_t); + data->pBind[bindIdx].buffer = data->tsData + rowIdx; + data->pBind[bindIdx].length = NULL; + data->pBind[bindIdx].is_null = NULL; + break; + case TSDB_DATA_TYPE_NCHAR: + data->pBind[bindIdx].buffer_length = gVarCharSize; + data->pBind[bindIdx].buffer = data->binaryData + rowIdx; + data->pBind[bindIdx].length = data->binaryLen; + data->pBind[bindIdx].is_null = data->isNull; + break; + case TSDB_DATA_TYPE_UTINYINT: + data->pBind[bindIdx].buffer_length = sizeof(uint8_t); + data->pBind[bindIdx].buffer = data->utinyData + rowIdx; + data->pBind[bindIdx].length = NULL; + data->pBind[bindIdx].is_null = data->isNull; + break; + case TSDB_DATA_TYPE_USMALLINT: + data->pBind[bindIdx].buffer_length = sizeof(uint16_t); + data->pBind[bindIdx].buffer = data->usmallData + rowIdx; + data->pBind[bindIdx].length = NULL; + data->pBind[bindIdx].is_null = data->isNull; + break; + case TSDB_DATA_TYPE_UINT: + data->pBind[bindIdx].buffer_length = sizeof(uint32_t); + data->pBind[bindIdx].buffer = data->uintData + rowIdx; + data->pBind[bindIdx].length = NULL; + data->pBind[bindIdx].is_null = data->isNull; + break; + case TSDB_DATA_TYPE_UBIGINT: + data->pBind[bindIdx].buffer_length = sizeof(uint64_t); + data->pBind[bindIdx].buffer = data->ubigData + rowIdx; + data->pBind[bindIdx].length = NULL; + data->pBind[bindIdx].is_null = data->isNull; + break; + default: + printf("invalid col type:%d", dataType); + exit(1); + } + + data->pBind[bindIdx].buffer_type = dataType; + data->pBind[bindIdx].num = gCurCase->bindRowNum; + + return 0; +} + + +int32_t prepareData(BindData *data) { + static int64_t tsData = 1591060628000; + uint64_t allRowNum = gCurCase->rowNum * gCurCase->tblNum; + + data->colNum = 0; + data->colTypes = taosMemoryCalloc(30, sizeof(int32_t)); + data->sql = taosMemoryCalloc(1, 1024); + data->pBind = taosMemoryCalloc((allRowNum/gCurCase->bindRowNum)*gCurCase->bindColNum, sizeof(TAOS_BIND_v2)); + data->tsData = taosMemoryMalloc(allRowNum * sizeof(int64_t)); + data->boolData = taosMemoryMalloc(allRowNum * sizeof(bool)); + data->tinyData = taosMemoryMalloc(allRowNum * sizeof(int8_t)); + data->utinyData = taosMemoryMalloc(allRowNum * sizeof(uint8_t)); + data->smallData = taosMemoryMalloc(allRowNum * sizeof(int16_t)); + data->usmallData = taosMemoryMalloc(allRowNum * sizeof(uint16_t)); + data->intData = taosMemoryMalloc(allRowNum * sizeof(int32_t)); + data->uintData = taosMemoryMalloc(allRowNum * sizeof(uint32_t)); + data->bigData = taosMemoryMalloc(allRowNum * sizeof(int64_t)); + data->ubigData = taosMemoryMalloc(allRowNum * sizeof(uint64_t)); + data->floatData = taosMemoryMalloc(allRowNum * sizeof(float)); + data->doubleData = taosMemoryMalloc(allRowNum * sizeof(double)); + data->binaryData = taosMemoryMalloc(allRowNum * gVarCharSize); + data->binaryLen = taosMemoryMalloc(allRowNum * sizeof(int32_t)); + if (gCurCase->bindNullNum) { + data->isNull = taosMemoryCalloc(allRowNum, sizeof(char)); } + for (int32_t i = 0; i < allRowNum; ++i) { + data->tsData[i] = tsData++; + data->boolData[i] = i % 2; + data->tinyData[i] = i; + data->utinyData[i] = i+1; + data->smallData[i] = i; + data->usmallData[i] = i+1; + data->intData[i] = i; + data->uintData[i] = i+1; + data->bigData[i] = i; + data->ubigData[i] = i+1; + data->floatData[i] = i; + data->doubleData[i] = i+1; + memset(data->binaryData + gVarCharSize * i, 'a'+i%26, gVarCharLen); + if (gCurCase->bindNullNum) { + data->isNull[i] = i % 2; + } + data->binaryLen[i] = gVarCharLen; + } - v.ts = 1591060628000; - v.v4 = 111; + for (int b = 0; b < (allRowNum/gCurCase->bindRowNum); b++) { + for (int c = 0; c < gCurCase->bindColNum; ++c) { + prepareColData(data, b*gCurCase->bindColNum+c, b*gCurCase->bindRowNum, c); + } + } - taos_stmt_bind_param(stmt, params); - taos_stmt_add_batch(stmt); + generateSql(data); + + return 0; +} + +void destroyData(BindData *data) { + taosMemoryFree(data->tsData); + taosMemoryFree(data->boolData); + taosMemoryFree(data->tinyData); + taosMemoryFree(data->utinyData); + taosMemoryFree(data->smallData); + taosMemoryFree(data->usmallData); + taosMemoryFree(data->intData); + taosMemoryFree(data->uintData); + taosMemoryFree(data->bigData); + taosMemoryFree(data->ubigData); + taosMemoryFree(data->floatData); + taosMemoryFree(data->doubleData); + taosMemoryFree(data->binaryData); + taosMemoryFree(data->binaryLen); + taosMemoryFree(data->isNull); + taosMemoryFree(data->pBind); +} + + +int insertAllCols(TAOS_STMT *stmt) { + BindData data = {0}; + prepareData(&data); + + printf("SQL: %s\n", data.sql); + + int code = taos_stmt_prepare(stmt, data.sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + int32_t bindTimes = gCurCase->rowNum/gCurCase->bindRowNum; + for (int32_t t = 0; t< gCurCase->tblNum; ++t) { + if (gCurCase->tblNum > 1) { + char buf[32]; + sprintf(buf, "t%d", t); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } + + for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { + printf("taos_stmt_bind_param error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + if (taos_stmt_add_batch(stmt)) { + printf("taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } + } if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement, error:%s.\n", taos_stmt_errstr(stmt)); + printf("taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + return 0; +} + + +int insertSpecifyCols(TAOS_STMT *stmt) { + BindData data = {0}; + prepareData(&data); + + printf("SQL: %s\n", data.sql); + + int code = taos_stmt_prepare(stmt, data.sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + int32_t bindTimes = gCurCase->rowNum/gCurCase->bindRowNum; + for (int32_t t = 0; t< gCurCase->tblNum; ++t) { + if (gCurCase->tblNum > 1) { + char buf[32]; + sprintf(buf, "t%d", t); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } + + for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { + printf("taos_stmt_bind_param error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + if (taos_stmt_add_batch(stmt)) { + printf("taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } + } + + if (taos_stmt_execute(stmt) != 0) { + printf("taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); exit(1); } @@ -365,7 +763,7 @@ int stmt_scol_func3(TAOS_STMT *stmt) { v.ts[i] = tts + i; } - unsigned long long starttime = taosGetTimestampUs(); + int64_t starttime = taosGetTimestampUs(); char *sql = "insert into ? (ts, v1,v2,f4,bin,bin2) values(?,?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); @@ -395,7 +793,7 @@ int stmt_scol_func3(TAOS_STMT *stmt) { ++id; } - unsigned long long endtime = taosGetTimestampUs(); + int64_t endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); taosMemoryFree(v.ts); @@ -487,7 +885,7 @@ int stmt_scol_func4(TAOS_STMT *stmt) { v.ts[i] = tts + i; } - unsigned long long starttime = taosGetTimestampUs(); + int64_t starttime = taosGetTimestampUs(); char *sql = "insert into ? (ts,b,v4,v8,f8) values(?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); @@ -518,7 +916,7 @@ int stmt_scol_func4(TAOS_STMT *stmt) { } } - unsigned long long endtime = taosGetTimestampUs(); + int64_t endtime = taosGetTimestampUs(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); taosMemoryFree(v.ts); @@ -530,6 +928,59 @@ int stmt_scol_func4(TAOS_STMT *stmt) { return 0; } + + + + +//1 tables 20 records +int stmt_scol_func5(TAOS_STMT *stmt) { + int32_t rowNum = gCurCase->rowNum; + int32_t bindRowNum = gCurCase->bindRowNum; + int32_t bindColNum = gCurCase->bindColNum; + int32_t nullNum = gCurCase->bindNullNum; + BindData data = {0}; + prepareData(&data); + + int64_t starttime = taosGetTimestampUs(); + + char *sql = "insert into ? (ts, v1,v2,f4,bin,bin2) values(?,?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + } + + int id = 0; + for (int b = 0; b < (rowNum/bindRowNum); b++) { + //for (int b = 0; b < 2; b++) { + for (int zz = 0; zz < 10; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); + } + + taos_stmt_bind_param_batch(stmt, data.pBind + b*bindColNum); + taos_stmt_add_batch(stmt); + } + + if (taos_stmt_execute(stmt) != 0) { + printf("failed to execute insert statement.\n"); + exit(1); + } + + ++id; + } + + int64_t endtime = taosGetTimestampUs(); + printf("insert total %d records, used %u seconds, avg:%u useconds\n", rowNum, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); + + destroyData(&data); + + return 0; +} + + #if 0 @@ -4261,12 +4712,13 @@ int stmt_funcb_sc3(TAOS_STMT *stmt) { } #endif -void check_result(TAOS *taos, char *tname, int printr, int expected) { + +void prepareCheckResultImpl(TAOS *taos, char *tname, int printr, int expected) { char sql[255] = "SELECT * FROM "; TAOS_RES *result; //FORCE NO PRINT - printr = 0; + //printr = 0; strcat(sql, tname); @@ -4307,6 +4759,20 @@ void check_result(TAOS *taos, char *tname, int printr, int expected) { } +void prepareCheckResult(TAOS *taos) { + char buf[32]; + for (int32_t t = 0; t< gCurCase->tblNum; ++t) { + if (gCurCase->tblNum > 1) { + sprintf(buf, "m%d", t); + } else { + sprintf(buf, "m%d", 0); + } + + prepareCheckResultImpl(taos, buf, 1, gCurCase->rowNum); + } +} + + //120table 60 record each table int sql_perf1(TAOS *taos) { @@ -4452,8 +4918,120 @@ int sql_s_perf1(TAOS *taos) { return 0; } +void generateCreateTableSQL(char *buf, int32_t tblIdx, int32_t colNum, int32_t *colList, bool stable) { + int32_t blen = 0; + blen = sprintf(buf, "create table %s%d ", (stable ? "st" : "t"), tblIdx); + if (stable) { + blen += sprintf(buf + blen, "tags ("); + for (int c = 0; c < colNum; ++c) { + switch (colList[c]) { + case TSDB_DATA_TYPE_BOOL: + blen += sprintf(buf + blen, "tbooldata bool"); + break; + case TSDB_DATA_TYPE_TINYINT: + blen += sprintf(buf + blen, "ttinydata tinyint"); + break; + case TSDB_DATA_TYPE_SMALLINT: + blen += sprintf(buf + blen, "tsmalldata smallint"); + break; + case TSDB_DATA_TYPE_INT: + blen += sprintf(buf + blen, "tintdata int"); + break; + case TSDB_DATA_TYPE_BIGINT: + blen += sprintf(buf + blen, "tbigdata bigint"); + break; + case TSDB_DATA_TYPE_FLOAT: + blen += sprintf(buf + blen, "tfloatdata float"); + break; + case TSDB_DATA_TYPE_DOUBLE: + blen += sprintf(buf + blen, "tdoubledata double"); + break; + case TSDB_DATA_TYPE_VARCHAR: + blen += sprintf(buf + blen, "tbinarydata binary(%d)", gVarCharSize); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + blen += sprintf(buf + blen, "tts ts"); + break; + case TSDB_DATA_TYPE_NCHAR: + blen += sprintf(buf + blen, "tnchardata nchar(%d)", gVarCharSize); + break; + case TSDB_DATA_TYPE_UTINYINT: + blen += sprintf(buf + blen, "tutinydata tinyint unsigned"); + break; + case TSDB_DATA_TYPE_USMALLINT: + blen += sprintf(buf + blen, "tusmalldata smallint unsigned"); + break; + case TSDB_DATA_TYPE_UINT: + blen += sprintf(buf + blen, "tuintdata int unsigned"); + break; + case TSDB_DATA_TYPE_UBIGINT: + blen += sprintf(buf + blen, "tubigdata bigint unsigned"); + break; + default: + printf("invalid col type:%d", colList[c]); + exit(1); + } + } -void prepare(TAOS *taos, int bigsize, int createChildTable) { + blen += sprintf(buf + blen, ")"); + } + + blen += sprintf(buf + blen, " ("); + + for (int c = 0; c < colNum; ++c) { + switch (colList[c]) { + case TSDB_DATA_TYPE_BOOL: + blen += sprintf(buf + blen, "booldata bool"); + break; + case TSDB_DATA_TYPE_TINYINT: + blen += sprintf(buf + blen, "tinydata tinyint"); + break; + case TSDB_DATA_TYPE_SMALLINT: + blen += sprintf(buf + blen, "smalldata smallint"); + break; + case TSDB_DATA_TYPE_INT: + blen += sprintf(buf + blen, "intdata int"); + break; + case TSDB_DATA_TYPE_BIGINT: + blen += sprintf(buf + blen, "bigdata bigint"); + break; + case TSDB_DATA_TYPE_FLOAT: + blen += sprintf(buf + blen, "floatdata float"); + break; + case TSDB_DATA_TYPE_DOUBLE: + blen += sprintf(buf + blen, "doubledata double"); + break; + case TSDB_DATA_TYPE_VARCHAR: + blen += sprintf(buf + blen, "binarydata binary(%d)", gVarCharSize); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + blen += sprintf(buf + blen, "ts ts"); + break; + case TSDB_DATA_TYPE_NCHAR: + blen += sprintf(buf + blen, "nchardata nchar(%d)", gVarCharSize); + break; + case TSDB_DATA_TYPE_UTINYINT: + blen += sprintf(buf + blen, "utinydata tinyint unsigned"); + break; + case TSDB_DATA_TYPE_USMALLINT: + blen += sprintf(buf + blen, "usmalldata smallint unsigned"); + break; + case TSDB_DATA_TYPE_UINT: + blen += sprintf(buf + blen, "uintdata int unsigned"); + break; + case TSDB_DATA_TYPE_UBIGINT: + blen += sprintf(buf + blen, "ubigdata bigint unsigned"); + break; + default: + printf("invalid col type:%d", colList[c]); + exit(1); + } + } + + blen += sprintf(buf + blen, ")"); +} + +void prepare(TAOS *taos, int32_t colNum, int32_t *colList, int autoCreate) { TAOS_RES *result; int code; @@ -4472,15 +5050,11 @@ void prepare(TAOS *taos, int bigsize, int createChildTable) { result = taos_query(taos, "use demo"); taos_free_result(result); - if (createChildTable) { + if (!autoCreate) { // create table - for (int i = 0 ; i < 300; i++) { + for (int i = 0 ; i < 10; i++) { char buf[1024]; - if (bigsize) { - sprintf(buf, "create table m%d (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin binary(40), bin2 binary(40))", i) ; - } else { - sprintf(buf, "create table m%d (ts timestamp, b int)", i) ; - } + generateCreateTableSQL(buf, i, colNum, colList, false); result = taos_query(taos, buf); code = taos_errno(result); if (code != 0) { @@ -4492,12 +5066,7 @@ void prepare(TAOS *taos, int bigsize, int createChildTable) { } } else { char buf[1024]; - if (bigsize) { - sprintf(buf, "create stable stb1 (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin binary(40), bin2 binary(40))" - " tags(id1 int, id2 bool, id3 tinyint, id4 smallint, id5 bigint, id6 float, id7 double, id8 binary(40), id9 nchar(40))") ; - } else { - sprintf(buf, "create stable stb1 (ts timestamp, b int) tags(id1 int, id2 bool, id3 tinyint, id4 smallint, id5 bigint, id6 float, id7 double, id8 binary(40), id9 nchar(40))") ; - } + generateCreateTableSQL(buf, 1, colNum, colList, true); result = taos_query(taos, buf); code = taos_errno(result); @@ -4511,65 +5080,29 @@ void prepare(TAOS *taos, int bigsize, int createChildTable) { } +void* runcase(TAOS *taos) { + TAOS_STMT *stmt = NULL; + for (int32_t i = 0; i < sizeof(gCase)/sizeof(gCase[0]); ++i) { + gCurCase = &gCase[i]; + printf("Case %d Begin\n", i); -void preparem(TAOS *taos, int bigsize, int idx) { - TAOS_RES *result; - int code; - char dbname[32],sql[255]; + for (int32_t n = 0; n < gCurCase->runTimes; ++n) { + prepare(taos, tListLen(gCurCase->colList), gCurCase->colList, gCurCase->autoCreate); + + stmt = taos_stmt_init(taos); + if (NULL == stmt) { + printf("taos_stmt_init failed, error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } - sprintf(dbname, "demo%d", idx); - sprintf(sql, "drop database %s", dbname); - + (*gCurCase->runFn)(stmt); - result = taos_query(taos, sql); - taos_free_result(result); - - sprintf(sql, "create database %s keep 36500", dbname); - result = taos_query(taos, sql); - code = taos_errno(result); - if (code != 0) { - printf("failed to create database, reason:%s\n", taos_errstr(result)); - taos_free_result(result); - exit(1); - } - taos_free_result(result); - - sprintf(sql, "use %s", dbname); - result = taos_query(taos, sql); - taos_free_result(result); - - // create table - for (int i = 0 ; i < 300; i++) { - char buf[1024]; - if (bigsize) { - sprintf(buf, "create table m%d (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin binary(40), bin2 binary(40))", i) ; - } else { - sprintf(buf, "create table m%d (ts timestamp, b int)", i) ; + prepareCheckResult(taos); } - result = taos_query(taos, buf); - code = taos_errno(result); - if (code != 0) { - printf("failed to create table, reason:%s\n", taos_errstr(result)); - taos_free_result(result); - exit(1); - } - taos_free_result(result); + printf("Case %d End\n", i); } -} - - - -//void runcase(TAOS *taos, int idx) { -void* runcase(void *par) { - T_par* tpar = (T_par *)par; - TAOS *taos = tpar->taos; - int idx = tpar->idx; - - TAOS_STMT *stmt; - - (void)idx; #if 0 prepare(taos, 0, 1); @@ -4636,7 +5169,7 @@ void* runcase(void *par) { -#if 1 +#if 0 prepare(taos, 1, 1); stmt = taos_stmt_init(taos); @@ -4655,9 +5188,8 @@ void* runcase(void *par) { #endif -#if 0 -#if 1 +#if 0 prepare(taos, 1, 1); stmt = taos_stmt_init(taos); @@ -4681,6 +5213,24 @@ void* runcase(void *par) { #endif +#if 0 + prepare(taos, 1, 1); + + stmt = taos_stmt_init(taos); + + printf("1t+2r+bm+specifycol start\n"); + stmt_scol_func5(stmt); + printf("1t+2r+bm+specifycol end\n"); + printf("check result start\n"); + check_result(taos, "m0", 1, 40); + printf("check result end\n"); + taos_stmt_close(stmt); + +#endif + + +#if 0 + #if 1 prepare(taos, 1, 1); @@ -5129,62 +5679,24 @@ void* runcase(void *par) { int main(int argc, char *argv[]) { - TAOS *taos[4]; + TAOS *taos = NULL; + srand(time(NULL)); + // connect to server if (argc < 2) { printf("please input server ip \n"); return 0; } - taos[0] = taos_connect(argv[1], "root", "taosdata", NULL, 0); + taos = taos_connect(argv[1], "root", "taosdata", NULL, 0); if (taos == NULL) { printf("failed to connect to db, reason:%s\n", taos_errstr(taos)); exit(1); } - taos[1] = taos_connect(argv[1], "root", "taosdata", NULL, 0); - if (taos == NULL) { - printf("failed to connect to db, reason:%s\n", taos_errstr(taos)); - exit(1); - } + runcase(taos); - taos[2] = taos_connect(argv[1], "root", "taosdata", NULL, 0); - if (taos == NULL) { - printf("failed to connect to db, reason:%s\n", taos_errstr(taos)); - exit(1); - } - - taos[3] = taos_connect(argv[1], "root", "taosdata", NULL, 0); - if (taos == NULL) { - printf("failed to connect to db, reason:%s\n", taos_errstr(taos)); - exit(1); - } - - pthread_t *pThreadList = (pthread_t *) taosMemoryCalloc(sizeof(pthread_t), 4); - - pthread_attr_t thattr; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - T_par par[4]; - - par[0].taos = taos[0]; - par[0].idx = 0; - par[1].taos = taos[1]; - par[1].idx = 1; - par[2].taos = taos[2]; - par[2].idx = 2; - par[3].taos = taos[3]; - par[3].idx = 3; - - pthread_create(&(pThreadList[0]), &thattr, runcase, (void *)&par[0]); - //pthread_create(&(pThreadList[1]), &thattr, runcase, (void *)&par[1]); - //pthread_create(&(pThreadList[2]), &thattr, runcase, (void *)&par[2]); - //pthread_create(&(pThreadList[3]), &thattr, runcase, (void *)&par[3]); - - while(1) { - sleep(1); - } return 0; } diff --git a/tests/script/api/makefile b/tests/script/api/makefile index 0c65ac6362..46a172cc3a 100644 --- a/tests/script/api/makefile +++ b/tests/script/api/makefile @@ -4,9 +4,9 @@ ROOT=./ TARGET=exe LFLAGS = '-Wl,-rpath,/usr/local/taos/driver/' -ltaos -lpthread -lm -lrt -CFLAGS = -O0 -g -Wall -Wno-deprecated -fPIC -Wno-unused-result -Wconversion \ +CFLAGS = -O0 -g -Wno-deprecated -fPIC -Wno-unused-result -Wconversion \ -Wno-char-subscripts -D_REENTRANT -Wno-format -D_REENTRANT -DLINUX \ - -Wno-unused-function -D_M_X64 -I/usr/local/taos/include -std=gnu99 + -Wno-unused-function -D_M_X64 -I/usr/local/taos/include -std=gnu99 -Wno-sign-conversion all: $(TARGET) From 2133a2de1d5ab0c761bf9fb9b85c0f5c139beba8 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 19 Apr 2022 20:30:06 +0800 Subject: [PATCH 23/47] refactor: add test casese for json compare logic --- include/common/tdataformat.h | 2 +- source/common/src/tdatablock.c | 21 ++- source/libs/scalar/src/sclvector.c | 4 +- source/libs/scalar/test/scalar/CMakeLists.txt | 3 +- .../libs/scalar/test/scalar/scalarTests.cpp | 143 +++++++----------- 5 files changed, 80 insertions(+), 93 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 2dd887ae74..a0f4351bbf 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -502,7 +502,7 @@ typedef struct { #define TD_KV_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t)) -#define kvRowLen(r) (*(TDRowLenT *)(r)) +#define kvRowLen(r) (*(uint16_t *)(r)) #define kvRowNCols(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(uint16_t))) #define kvRowSetLen(r, len) kvRowLen(r) = (len) #define kvRowSetNCols(r, n) kvRowNCols(r) = (n) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 14b7fe5d0e..5bdf7ddb85 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -113,14 +113,27 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con int32_t type = pColumnInfoData->info.type; if (IS_VAR_DATA_TYPE(type)) { + int32_t dataLen = varDataTLen(pData); + if(type == TSDB_DATA_TYPE_JSON) { + if(*pData == TSDB_DATA_TYPE_NULL) { + dataLen = 0; + }else if(*pData == TSDB_DATA_TYPE_NCHAR) { + dataLen = varDataTLen(pData+CHAR_BYTES); + }else if(*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) { + dataLen = LONG_BYTES; + }else if(*pData == TSDB_DATA_TYPE_BOOL) { + dataLen = CHAR_BYTES; + } + dataLen += CHAR_BYTES; + } SVarColAttr* pAttr = &pColumnInfoData->varmeta; - if (pAttr->allocLen < pAttr->length + varDataTLen(pData)) { + if (pAttr->allocLen < pAttr->length + dataLen) { uint32_t newSize = pAttr->allocLen; if (newSize == 0) { newSize = 8; } - while (newSize < pAttr->length + varDataTLen(pData)) { + while (newSize < pAttr->length + dataLen) { newSize = newSize * 1.5; } @@ -136,8 +149,8 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con uint32_t len = pColumnInfoData->varmeta.length; pColumnInfoData->varmeta.offset[currentRow] = len; - memcpy(pColumnInfoData->pData + len, pData, varDataTLen(pData)); - pColumnInfoData->varmeta.length += varDataTLen(pData); + memcpy(pColumnInfoData->pData + len, pData, dataLen); + pColumnInfoData->varmeta.length += dataLen; } else { memcpy(pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow, pData, pColumnInfoData->info.bytes); } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 7ad200befc..c2eafedf9b 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -774,7 +774,7 @@ char *getJsonValue(char *json, char *key){ //todo if(i == 0 && *data == TSDB_DATA_TYPE_NULL){ return NULL; } - if(strncmp(key, varDataVal(data), varDataLen(data)) == 0){ + if(memcmp(key, data, varDataTLen(data)) == 0){ return data + varDataTLen(data); } } @@ -803,7 +803,7 @@ void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO pOutputCol->hasNull = true; continue; } - colDataAppend(pOutputCol, i, pLeftData, false); + colDataAppend(pOutputCol, i, value, false); } } diff --git a/source/libs/scalar/test/scalar/CMakeLists.txt b/source/libs/scalar/test/scalar/CMakeLists.txt index 6cbac1e25d..03418d17ac 100644 --- a/source/libs/scalar/test/scalar/CMakeLists.txt +++ b/source/libs/scalar/test/scalar/CMakeLists.txt @@ -8,11 +8,12 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) ADD_EXECUTABLE(scalarTest ${SOURCE_LIST}) TARGET_LINK_LIBRARIES( scalarTest - PUBLIC os util common gtest qcom function nodes scalar + PUBLIC os util common gtest qcom function nodes scalar parser ) TARGET_INCLUDE_DIRECTORIES( scalarTest PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar/" + PUBLIC "${TD_SOURCE_DIR}/source/libs/parser/inc" PRIVATE "${TD_SOURCE_DIR}/source/libs/scalar/inc" ) diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 975effc63a..8049e80590 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -125,56 +125,6 @@ void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) { *pNode = (SNode *)vnode; } -void scltMakeJsonColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { - SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_COLUMN); - SColumnNode *rnode = (SColumnNode *)node; - rnode->node.resType.type = dataType; - rnode->node.resType.bytes = dataBytes; - rnode->dataBlockId = 0; - - if (NULL == *block) { - SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); - res->info.numOfCols = 3; - res->info.rows = rowNum; - res->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData)); - for (int32_t i = 0; i < 2; ++i) { - SColumnInfoData idata = {{0}}; - idata.info.type = TSDB_DATA_TYPE_NULL; - idata.info.bytes = 10; - idata.info.colId = i + 1; - - int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)taosMemoryCalloc(1, size); - taosArrayPush(res->pDataBlock, &idata); - } - - SColumnInfoData idata = {{0}}; - idata.info.type = dataType; - idata.info.bytes = dataBytes; - idata.info.colId = 3; - int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)taosMemoryCalloc(1, size); - taosArrayPush(res->pDataBlock, &idata); - - blockDataEnsureCapacity(res, rowNum); - - SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); - for (int32_t i = 0; i < rowNum; ++i) { - uint32_t len = pColumnInfoData->varmeta.length; - pColumnInfoData->varmeta.offset[i] = len; - - memcpy(pColumnInfoData->pData + len, value, varDataLen(value)); - pColumnInfoData->varmeta.length += varDataLen(value); - } - - rnode->slotId = 2; - rnode->colId = 3; - - *block = res; - } - - *pNode = (SNode *)rnode; -} void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_COLUMN); @@ -960,54 +910,77 @@ TEST(constantTest, greater_and_lower) { nodesDestroyNode(res); } -TEST(columnTest, smallint_value_add_json_column) { - scltInitLogFile(); +void makeJsonArrow(SSDataBlock **src, SNode **opNode, void *json, char *key){ + char key[32] = {0}; + memcpy(varDataVal(key), key, strlen(key)); + varDataLen(key) = strlen(key); SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; - int32_t leftv = 1; - char *rightv= "{"k1":4,"k2":"hello","k3":null,"k4":true,"k5":5.44}"; - char key[10] = {0}; - memcpy(varDataVal(key), "k1", 2); - varDataLen(key) = 2; + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, key); + scltMakeColumnNode(&pLeft, src, TSDB_DATA_TYPE_JSON, varDataLen(json), 1, json); + scltMakeOpNode(opNode, OP_TYPE_JSON_GET_VALUE, TSDB_DATA_TYPE_JSON, pLeft, pRight); +} + +void makeOperator(SNode **opNode, SArray *blockList, int32_t resType, EOperatorType opType, int32_t rightType, void *rightData){ + SNode *right = NULL; + scltMakeValueNode(&right, rightType, rightData); + scltMakeOpNode(opNode, opType, resType, *opNode, right); + + SColumnInfo colInfo = createColumnInfo(1, resType, tDataTypes[resType].bytes); + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, 1, &colInfo); + scltMakeTargetNode(opNode, dataBlockId, slotId, *opNode); +} + +void makeCalculate(void *json, void *key, void *value, double exceptValue, EOperatorType opType){ + SArray *blockList = taosArrayInit(2, POINTER_BYTES); + SSDataBlock *src = NULL; + SNode *opNode = NULL; + + makeJsonArrow(&src, &opNode, row, key); + taosArrayPush(blockList, &src); + + makeOperator(&opNode, blockList, TSDB_DATA_TYPE_DOUBLE, opType, TSDB_DATA_TYPE_INT, value); + + int32_t code = scalarCalculate(opNode, blockList, NULL); + ASSERT_EQ(code, 0); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, 1); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_DOUBLE); + ASSERT_EQ(*((double *)colDataGetData(column, 0)), exceptValue); + + taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); +} + +TEST(columnTest, int_value_add_json_column) { + scltInitLogFile(); + char *rightv= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44}"; + char *key = "k1"; SKVRowBuilder kvRowBuilder; tdInitKVRowBuilder(&kvRowBuilder); parseJsontoTagData(rightv, &kvRowBuilder, NULL, 0); SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); - double eRes[5] = {1.0, -4, -3, 24, 101}; + int32_t leftv = 1; + makeCalculate(row, key, &leftv, 5.0, OP_TYPE_ADD); - SSDataBlock *src = NULL; - int32_t rowNum = 1; + leftv = 8; + makeCalculate(row, key, &leftv, -4, OP_TYPE_SUB); - scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, key); - scltMakeJsonColumnNode(&pLeft, &src, TSDB_DATA_TYPE_JSON, varDataLen(row), rowNum, row); - scltMakeOpNode(&opNode, OP_TYPE_JSON_GET_VALUE, TSDB_DATA_TYPE_JSON, pLeft, pRight); + leftv = 2; + makeCalculate(row, key, &leftv, 8.0, OP_TYPE_MULTI); - SNode *right = NULL; - scltMakeValueNode(&right, TSDB_DATA_TYPE_INT, &leftv); - scltMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, opNode, right); + leftv = 2; + makeCalculate(row, key, &leftv, 2.0, OP_TYPE_DIV); - SArray *blockList = taosArrayInit(2, POINTER_BYTES); - taosArrayPush(blockList, &src); - SColumnInfo colInfo = createColumnInfo(1, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); - int16_t dataBlockId = 0, slotId = 0; - scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo); - scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); + leftv = 3; + makeCalculate(row, key, &leftv, 1.0, OP_TYPE_MODE); - int32_t code = scalarCalculate(opNode, blockList, NULL); - ASSERT_EQ(code, 0); - - SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); - ASSERT_EQ(res->info.rows, rowNum); - SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); - ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_DOUBLE); - for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((double *)colDataGetData(column, i)), eRes[i]); - } - - taosArrayDestroyEx(blockList, scltFreeDataBlock); - nodesDestroyNode(opNode); + makeCalculate(row, key, &leftv, -4, OP_TYPE_MINUS); } TEST(columnTest, smallint_value_add_int_column) { From 743ff696b880d09271bc32f4369d74302b54ae5f Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 20 Apr 2022 08:58:29 +0800 Subject: [PATCH 24/47] stmt --- tests/script/api/batchprepare.c | 70 ++++++++++++++++++++------------- 1 file changed, 43 insertions(+), 27 deletions(-) diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 0740560f32..4b259ceafe 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -34,8 +34,8 @@ typedef struct { int32_t gVarCharSize = 10; int32_t gVarCharLen = 5; -int insertAllCols(TAOS_STMT *stmt); -int insertSpecifyCols(TAOS_STMT *stmt); +int insertMBSETest(TAOS_STMT *stmt); +int insertMBMETest(TAOS_STMT *stmt); int32_t shortColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_INT}; int32_t longColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_BOOL, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_UTINYINT, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_USMALLINT, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_UINT, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_UBIGINT, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_BINARY, TSDB_DATA_TYPE_NCHAR}; @@ -53,16 +53,17 @@ typedef struct { int32_t tblNum; int32_t rowNum; int32_t bindRowNum; - int32_t bindColNum; + int32_t bindColNum; // equal colNum in full column case int32_t bindNullNum; int32_t runTimes; } CaseCfg; CaseCfg gCase[] = { -// {"insert:all columns", tListLen(shortColList), shortColList, false, true, insertAllCols, 1, 10, 10, 2, 0, 10}, -// {"insert:all columns", tListLen(shortColList), shortColList, false, true, insertAllCols, 10, 100, 10, 2, 0, 10}, - {"insert:all columns", tListLen(longColList), longColList, false, true, insertAllCols, 10, 10, 2, 14, 0, 1}, -// {"insert:all columns", tListLen(longColList), longColList, false, false, insertSpecifyCols, 10, 10, 2, 6, 0, 1}, +// {"insert:MBSE", tListLen(shortColList), shortColList, false, true, insertMBSETest, 1, 10, 10, 0, 0, 10}, +// {"insert:MBSE", tListLen(shortColList), shortColList, false, true, insertMBSETest, 10, 100, 10, 0, 0, 10}, +// {"insert:MBSE", tListLen(longColList), longColList, false, true, insertMBSETest, 10, 10, 2, 0, 0, 1}, +// {"insert:MBSE", tListLen(longColList), longColList, false, false, insertMBSETest, 10, 10, 2, 6, 0, 1}, + {"insert:MBSE", tListLen(longColList), longColList, false, false, insertMBMETest, 10, 10, 2, 6, 0, 1}, }; CaseCfg *gCurCase = NULL; @@ -111,7 +112,7 @@ bool colExists(TAOS_BIND_v2* pBind, int32_t dataType) { } } -void generateSql(BindData *data) { +void generateInsertSQL(BindData *data) { int32_t len = sprintf(data->sql, "insert into %s ", (gCurCase->tblNum > 1 ? "? " : "m0 ")); if (!gCurCase->fullCol) { len += sprintf(data->sql + len, "("); @@ -184,12 +185,13 @@ void generateSql(BindData *data) { void generateDataType(BindData *data, int32_t bindIdx, int32_t colIdx, int32_t *dataType) { if (bindIdx < gCurCase->bindColNum) { - if (colIdx) { - if (!gCurCase->multiCol) { - *dataType = TSDB_DATA_TYPE_INT; - return; - } - + if (gCurCase->fullCol) { + *dataType = gCurCase->colList[bindIdx]; + return; + } else if (0 == colIdx) { + *dataType = TSDB_DATA_TYPE_TIMESTAMP; + return; + } else { while (true) { *dataType = rand() % (TSDB_DATA_TYPE_MAX - 1) + 1; if (*dataType == TSDB_DATA_TYPE_JSON || *dataType == TSDB_DATA_TYPE_DECIMAL @@ -260,7 +262,7 @@ int32_t prepareColData(BindData *data, int32_t bindIdx, int32_t rowIdx, int32_t break; case TSDB_DATA_TYPE_VARCHAR: data->pBind[bindIdx].buffer_length = gVarCharSize; - data->pBind[bindIdx].buffer = data->binaryData + rowIdx; + data->pBind[bindIdx].buffer = data->binaryData + rowIdx * gVarCharSize; data->pBind[bindIdx].length = data->binaryLen; data->pBind[bindIdx].is_null = data->isNull; break; @@ -272,7 +274,7 @@ int32_t prepareColData(BindData *data, int32_t bindIdx, int32_t rowIdx, int32_t break; case TSDB_DATA_TYPE_NCHAR: data->pBind[bindIdx].buffer_length = gVarCharSize; - data->pBind[bindIdx].buffer = data->binaryData + rowIdx; + data->pBind[bindIdx].buffer = data->binaryData + rowIdx * gVarCharSize; data->pBind[bindIdx].length = data->binaryLen; data->pBind[bindIdx].is_null = data->isNull; break; @@ -364,7 +366,7 @@ int32_t prepareData(BindData *data) { } } - generateSql(data); + generateInsertSQL(data); return 0; } @@ -389,7 +391,7 @@ void destroyData(BindData *data) { } -int insertAllCols(TAOS_STMT *stmt) { +int insertMBSETest(TAOS_STMT *stmt) { BindData data = {0}; prepareData(&data); @@ -435,7 +437,7 @@ int insertAllCols(TAOS_STMT *stmt) { } -int insertSpecifyCols(TAOS_STMT *stmt) { +int insertMBMETest(TAOS_STMT *stmt) { BindData data = {0}; prepareData(&data); @@ -470,12 +472,13 @@ int insertSpecifyCols(TAOS_STMT *stmt) { exit(1); } } + + if (taos_stmt_execute(stmt) != 0) { + printf("taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } } - if (taos_stmt_execute(stmt) != 0) { - printf("taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); - exit(1); - } return 0; } @@ -4763,9 +4766,9 @@ void prepareCheckResult(TAOS *taos) { char buf[32]; for (int32_t t = 0; t< gCurCase->tblNum; ++t) { if (gCurCase->tblNum > 1) { - sprintf(buf, "m%d", t); + sprintf(buf, "t%d", t); } else { - sprintf(buf, "m%d", 0); + sprintf(buf, "t%d", 0); } prepareCheckResultImpl(taos, buf, 1, gCurCase->rowNum); @@ -4924,6 +4927,9 @@ void generateCreateTableSQL(char *buf, int32_t tblIdx, int32_t colNum, int32_t * if (stable) { blen += sprintf(buf + blen, "tags ("); for (int c = 0; c < colNum; ++c) { + if (c > 0) { + blen += sprintf(buf + blen, ","); + } switch (colList[c]) { case TSDB_DATA_TYPE_BOOL: blen += sprintf(buf + blen, "tbooldata bool"); @@ -4979,6 +4985,10 @@ void generateCreateTableSQL(char *buf, int32_t tblIdx, int32_t colNum, int32_t * blen += sprintf(buf + blen, " ("); for (int c = 0; c < colNum; ++c) { + if (c > 0) { + blen += sprintf(buf + blen, ","); + } + switch (colList[c]) { case TSDB_DATA_TYPE_BOOL: blen += sprintf(buf + blen, "booldata bool"); @@ -5005,7 +5015,7 @@ void generateCreateTableSQL(char *buf, int32_t tblIdx, int32_t colNum, int32_t * blen += sprintf(buf + blen, "binarydata binary(%d)", gVarCharSize); break; case TSDB_DATA_TYPE_TIMESTAMP: - blen += sprintf(buf + blen, "ts ts"); + blen += sprintf(buf + blen, "ts timestamp"); break; case TSDB_DATA_TYPE_NCHAR: blen += sprintf(buf + blen, "nchardata nchar(%d)", gVarCharSize); @@ -5029,6 +5039,8 @@ void generateCreateTableSQL(char *buf, int32_t tblIdx, int32_t colNum, int32_t * } blen += sprintf(buf + blen, ")"); + + printf("Create SQL:%s\n", buf); } void prepare(TAOS *taos, int32_t colNum, int32_t *colList, int autoCreate) { @@ -5087,8 +5099,12 @@ void* runcase(TAOS *taos) { gCurCase = &gCase[i]; printf("Case %d Begin\n", i); + if (gCurCase->fullCol) { + gCurCase->bindColNum = gCurCase->colNum; + } + for (int32_t n = 0; n < gCurCase->runTimes; ++n) { - prepare(taos, tListLen(gCurCase->colList), gCurCase->colList, gCurCase->autoCreate); + prepare(taos, gCurCase->colNum, gCurCase->colList, gCurCase->autoCreate); stmt = taos_stmt_init(taos); if (NULL == stmt) { From da4379853d3d335412f16658bfa13e34604cadf8 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 20 Apr 2022 09:12:59 +0800 Subject: [PATCH 25/47] stmt --- include/libs/planner/planner.h | 1 - source/libs/parser/src/parInsert.c | 1 - tests/script/api/batchprepare.c | 4 ++-- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index f343295c56..eba70b8098 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -45,7 +45,6 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo // @pSource one execution location of this group of datasource subplans int32_t qSetSubplanExecutionNode(SSubplan* pSubplan, int32_t groupId, SDownstreamSourceNode* pSource); -typedef TAOS_MULTI_BIND TAOS_BIND_v2; // todo remove int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_BIND_v2* pParams); // Convert to subplan to string for the scheduler to send to the executor diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 60f5ae4283..b6d835846c 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1009,7 +1009,6 @@ static int32_t parseValuesClause(SInsertParseContext* pCxt, STableDataBlocks* da } void destroyCreateSubTbReq(SVCreateTbReq* pReq) { - taosMemoryFreeClear(pReq->dbFName); taosMemoryFreeClear(pReq->name); taosMemoryFreeClear(pReq->ctbCfg.pTag); } diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 4b259ceafe..9936c0572c 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -63,7 +63,7 @@ CaseCfg gCase[] = { // {"insert:MBSE", tListLen(shortColList), shortColList, false, true, insertMBSETest, 10, 100, 10, 0, 0, 10}, // {"insert:MBSE", tListLen(longColList), longColList, false, true, insertMBSETest, 10, 10, 2, 0, 0, 1}, // {"insert:MBSE", tListLen(longColList), longColList, false, false, insertMBSETest, 10, 10, 2, 6, 0, 1}, - {"insert:MBSE", tListLen(longColList), longColList, false, false, insertMBMETest, 10, 10, 2, 6, 0, 1}, + {"insert:MBME", tListLen(longColList), longColList, false, false, insertMBMETest, 10, 10, 2, 6, 0, 1}, }; CaseCfg *gCurCase = NULL; @@ -5097,7 +5097,7 @@ void* runcase(TAOS *taos) { for (int32_t i = 0; i < sizeof(gCase)/sizeof(gCase[0]); ++i) { gCurCase = &gCase[i]; - printf("Case %d Begin\n", i); + printf("Case %d - %s Begin\n", i, gCurCase->caseDesc); if (gCurCase->fullCol) { gCurCase->bindColNum = gCurCase->colNum; From 7a740912ef64f84ef02a008b31d9bf35985e609c Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 20 Apr 2022 09:36:47 +0800 Subject: [PATCH 26/47] bug fix --- source/common/src/trow.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index a1a2d236f9..6685fdf2a7 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -496,8 +496,9 @@ static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols SKvRowIdx *pIdx = tdKvRowColIdxAt(pRow, rcol); int16_t colIdx = -1; if (pIdx) { - colIdx = POINTER_DISTANCE(pRow->data, pIdx) / sizeof(SKvRowIdx); + colIdx = POINTER_DISTANCE(pIdx, TD_ROW_COL_IDX(pRow)) / sizeof(SKvRowIdx); } + TASSERT(colIdx >= 0); SCellVal sVal = {0}; if (pIdx->colId == pDataCol->colId) { if (tdGetKvRowValOfCol(&sVal, pRow, pBitmap, pIdx->offset, colIdx) < 0) { From 5acc77393f4156296e00ceb73c1eb0226140014c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 20 Apr 2022 09:39:55 +0800 Subject: [PATCH 27/47] stmt --- tests/script/api/batchprepare.c | 124 +++++++++++++++++++++++++++++--- 1 file changed, 114 insertions(+), 10 deletions(-) diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 9936c0572c..e25e3d3526 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -34,8 +34,10 @@ typedef struct { int32_t gVarCharSize = 10; int32_t gVarCharLen = 5; -int insertMBSETest(TAOS_STMT *stmt); -int insertMBMETest(TAOS_STMT *stmt); +int insertMBSETest1(TAOS_STMT *stmt); +int insertMBSETest2(TAOS_STMT *stmt); +int insertMBMETest1(TAOS_STMT *stmt); +int insertMBMETest2(TAOS_STMT *stmt); int32_t shortColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_INT}; int32_t longColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_BOOL, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_UTINYINT, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_USMALLINT, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_UINT, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_UBIGINT, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_BINARY, TSDB_DATA_TYPE_NCHAR}; @@ -59,11 +61,17 @@ typedef struct { } CaseCfg; CaseCfg gCase[] = { -// {"insert:MBSE", tListLen(shortColList), shortColList, false, true, insertMBSETest, 1, 10, 10, 0, 0, 10}, -// {"insert:MBSE", tListLen(shortColList), shortColList, false, true, insertMBSETest, 10, 100, 10, 0, 0, 10}, -// {"insert:MBSE", tListLen(longColList), longColList, false, true, insertMBSETest, 10, 10, 2, 0, 0, 1}, -// {"insert:MBSE", tListLen(longColList), longColList, false, false, insertMBSETest, 10, 10, 2, 6, 0, 1}, - {"insert:MBME", tListLen(longColList), longColList, false, false, insertMBMETest, 10, 10, 2, 6, 0, 1}, +#if 0 + {"insert:MBSE1", tListLen(shortColList), shortColList, false, true, insertMBSETest1, 1, 10, 10, 0, 0, 10}, + {"insert:MBSE1", tListLen(shortColList), shortColList, false, true, insertMBSETest1, 10, 100, 10, 0, 0, 10}, + {"insert:MBSE1", tListLen(longColList), longColList, false, true, insertMBSETest1, 10, 10, 2, 0, 0, 1}, + {"insert:MBSE1", tListLen(longColList), longColList, false, false, insertMBSETest1, 10, 10, 2, 6, 0, 1}, +#endif +// {"insert:MBSE2", tListLen(longColList), longColList, false, true, insertMBSETest2, 10, 10, 2, 0, 0, 1}, +// {"insert:MBSE2", tListLen(longColList), longColList, false, false, insertMBSETest2, 10, 10, 2, 6, 0, 1}, +// {"insert:MBME1", tListLen(longColList), longColList, false, false, insertMBMETest1, 10, 10, 2, 6, 0, 1}, + {"insert:MBME2", tListLen(longColList), longColList, false, false, insertMBMETest2, 10, 10, 2, 6, 0, 1}, + }; CaseCfg *gCurCase = NULL; @@ -113,7 +121,7 @@ bool colExists(TAOS_BIND_v2* pBind, int32_t dataType) { } void generateInsertSQL(BindData *data) { - int32_t len = sprintf(data->sql, "insert into %s ", (gCurCase->tblNum > 1 ? "? " : "m0 ")); + int32_t len = sprintf(data->sql, "insert into %s ", (gCurCase->tblNum > 1 ? "? " : "t0 ")); if (!gCurCase->fullCol) { len += sprintf(data->sql + len, "("); for (int c = 0; c < gCurCase->bindColNum; ++c) { @@ -391,7 +399,8 @@ void destroyData(BindData *data) { } -int insertMBSETest(TAOS_STMT *stmt) { +/* prepare [settbname [bind add]] exec */ +int insertMBSETest1(TAOS_STMT *stmt) { BindData data = {0}; prepareData(&data); @@ -437,7 +446,55 @@ int insertMBSETest(TAOS_STMT *stmt) { } -int insertMBMETest(TAOS_STMT *stmt) { +/* prepare [settbname bind add] exec */ +int insertMBSETest2(TAOS_STMT *stmt) { + BindData data = {0}; + prepareData(&data); + + printf("SQL: %s\n", data.sql); + + int code = taos_stmt_prepare(stmt, data.sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + int32_t bindTimes = gCurCase->rowNum/gCurCase->bindRowNum; + + for (int32_t b = 0; b tblNum; ++t) { + if (gCurCase->tblNum > 1) { + char buf[32]; + sprintf(buf, "t%d", t); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } + + if (taos_stmt_bind_param_batch(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { + printf("taos_stmt_bind_param error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + if (taos_stmt_add_batch(stmt)) { + printf("taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } + } + + if (taos_stmt_execute(stmt) != 0) { + printf("taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + return 0; +} + +/* prepare [settbname [bind add] exec] */ +int insertMBMETest1(TAOS_STMT *stmt) { BindData data = {0}; prepareData(&data); @@ -483,6 +540,53 @@ int insertMBMETest(TAOS_STMT *stmt) { return 0; } +/* prepare [settbname [bind add exec]] */ +int insertMBMETest2(TAOS_STMT *stmt) { + BindData data = {0}; + prepareData(&data); + + printf("SQL: %s\n", data.sql); + + int code = taos_stmt_prepare(stmt, data.sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + int32_t bindTimes = gCurCase->rowNum/gCurCase->bindRowNum; + for (int32_t t = 0; t< gCurCase->tblNum; ++t) { + if (gCurCase->tblNum > 1) { + char buf[32]; + sprintf(buf, "t%d", t); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } + + for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { + printf("taos_stmt_bind_param error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + if (taos_stmt_add_batch(stmt)) { + printf("taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + if (taos_stmt_execute(stmt) != 0) { + printf("taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } + } + + + return 0; +} + int stmt_scol_func1(TAOS_STMT *stmt) { struct { From 9c178b01e814b97b4abfdfe06bfc93ccd383a42a Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 20 Apr 2022 10:06:49 +0800 Subject: [PATCH 28/47] stmt --- tests/script/api/batchprepare.c | 142 +++++++++++++++++++++++++++++--- 1 file changed, 130 insertions(+), 12 deletions(-) diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index e25e3d3526..bdaa4f7172 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -38,6 +38,8 @@ int insertMBSETest1(TAOS_STMT *stmt); int insertMBSETest2(TAOS_STMT *stmt); int insertMBMETest1(TAOS_STMT *stmt); int insertMBMETest2(TAOS_STMT *stmt); +int insertMBMETest3(TAOS_STMT *stmt); +int insertMBMETest4(TAOS_STMT *stmt); int32_t shortColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_INT}; int32_t longColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_BOOL, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_UTINYINT, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_USMALLINT, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_UINT, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_UBIGINT, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_BINARY, TSDB_DATA_TYPE_NCHAR}; @@ -61,16 +63,26 @@ typedef struct { } CaseCfg; CaseCfg gCase[] = { -#if 0 - {"insert:MBSE1", tListLen(shortColList), shortColList, false, true, insertMBSETest1, 1, 10, 10, 0, 0, 10}, - {"insert:MBSE1", tListLen(shortColList), shortColList, false, true, insertMBSETest1, 10, 100, 10, 0, 0, 10}, - {"insert:MBSE1", tListLen(longColList), longColList, false, true, insertMBSETest1, 10, 10, 2, 0, 0, 1}, - {"insert:MBSE1", tListLen(longColList), longColList, false, false, insertMBSETest1, 10, 10, 2, 6, 0, 1}, -#endif -// {"insert:MBSE2", tListLen(longColList), longColList, false, true, insertMBSETest2, 10, 10, 2, 0, 0, 1}, -// {"insert:MBSE2", tListLen(longColList), longColList, false, false, insertMBSETest2, 10, 10, 2, 6, 0, 1}, -// {"insert:MBME1", tListLen(longColList), longColList, false, false, insertMBMETest1, 10, 10, 2, 6, 0, 1}, - {"insert:MBME2", tListLen(longColList), longColList, false, false, insertMBMETest2, 10, 10, 2, 6, 0, 1}, +// {"insert:MBSE1-FULL", tListLen(shortColList), shortColList, false, true, insertMBSETest1, 1, 10, 10, 0, 0, 10}, +// {"insert:MBSE1-FULL", tListLen(shortColList), shortColList, false, true, insertMBSETest1, 10, 100, 10, 0, 0, 10}, +// {"insert:MBSE1-FULL", tListLen(longColList), longColList, false, true, insertMBSETest1, 10, 10, 2, 0, 0, 1}, +// {"insert:MBSE1-C012", tListLen(longColList), longColList, false, false, insertMBSETest1, 10, 10, 2, 12, 0, 1}, +// {"insert:MBSE1-C002", tListLen(longColList), longColList, false, false, insertMBSETest1, 10, 10, 2, 2, 0, 1}, +// {"insert:MBSE2-FULL", tListLen(longColList), longColList, false, true, insertMBSETest2, 10, 10, 2, 0, 0, 1}, +// {"insert:MBSE2-C012", tListLen(longColList), longColList, false, false, insertMBSETest2, 10, 10, 2, 12, 0, 1}, +// {"insert:MBSE2-C002", tListLen(longColList), longColList, false, false, insertMBSETest2, 10, 10, 2, 2, 0, 1}, +// {"insert:MBME1-FULL", tListLen(longColList), longColList, false, true, insertMBMETest1, 10, 10, 2, 0, 0, 1}, +// {"insert:MBME1-C012", tListLen(longColList), longColList, false, false, insertMBMETest1, 10, 10, 2, 12, 0, 1}, +// {"insert:MBME1-C002", tListLen(longColList), longColList, false, false, insertMBMETest1, 10, 10, 2, 2, 0, 1}, +// {"insert:MBME2-FULL", tListLen(longColList), longColList, false, true, insertMBMETest2, 10, 10, 2, 0, 0, 1}, +// {"insert:MBME2-C012", tListLen(longColList), longColList, false, false, insertMBMETest2, 10, 10, 2, 12, 0, 1}, +// {"insert:MBME2-C002", tListLen(longColList), longColList, false, false, insertMBMETest2, 10, 10, 2, 2, 0, 1}, +// {"insert:MBME3-FULL", tListLen(longColList), longColList, false, true, insertMBMETest3, 10, 10, 2, 0, 0, 1}, +// {"insert:MBME3-C012", tListLen(longColList), longColList, false, false, insertMBMETest3, 10, 10, 2, 12, 0, 1}, +// {"insert:MBME3-C002", tListLen(longColList), longColList, false, false, insertMBMETest3, 10, 10, 2, 2, 0, 1}, +// {"insert:MBME4-FULL", tListLen(longColList), longColList, false, true, insertMBMETest4, 10, 10, 2, 0, 0, 1}, +// {"insert:MBME4-C012", tListLen(longColList), longColList, false, false, insertMBMETest4, 10, 10, 2, 12, 0, 1}, + {"insert:MBME4-C002", tListLen(longColList), longColList, false, false, insertMBMETest4, 10, 10, 2, 2, 0, 1}, }; @@ -587,6 +599,111 @@ int insertMBMETest2(TAOS_STMT *stmt) { return 0; } +/* prepare [settbname [settbname bind add exec]] */ +int insertMBMETest3(TAOS_STMT *stmt) { + BindData data = {0}; + prepareData(&data); + + printf("SQL: %s\n", data.sql); + + int code = taos_stmt_prepare(stmt, data.sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + int32_t bindTimes = gCurCase->rowNum/gCurCase->bindRowNum; + for (int32_t t = 0; t< gCurCase->tblNum; ++t) { + if (gCurCase->tblNum > 1) { + char buf[32]; + sprintf(buf, "t%d", t); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } + + for (int32_t b = 0; b tblNum > 1) { + char buf[32]; + sprintf(buf, "t%d", t); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } + + if (taos_stmt_bind_param_batch(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { + printf("taos_stmt_bind_param error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + if (taos_stmt_add_batch(stmt)) { + printf("taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + if (taos_stmt_execute(stmt) != 0) { + printf("taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } + } + + + return 0; +} + + +/* prepare [settbname bind add exec] */ +int insertMBMETest4(TAOS_STMT *stmt) { + BindData data = {0}; + prepareData(&data); + + printf("SQL: %s\n", data.sql); + + int code = taos_stmt_prepare(stmt, data.sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + int32_t bindTimes = gCurCase->rowNum/gCurCase->bindRowNum; + + for (int32_t b = 0; b tblNum; ++t) { + if (gCurCase->tblNum > 1) { + char buf[32]; + sprintf(buf, "t%d", t); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } + + if (taos_stmt_bind_param_batch(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { + printf("taos_stmt_bind_param error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + if (taos_stmt_add_batch(stmt)) { + printf("taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + if (taos_stmt_execute(stmt) != 0) { + printf("taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } + } + + return 0; +} + int stmt_scol_func1(TAOS_STMT *stmt) { struct { @@ -5201,7 +5318,7 @@ void* runcase(TAOS *taos) { for (int32_t i = 0; i < sizeof(gCase)/sizeof(gCase[0]); ++i) { gCurCase = &gCase[i]; - printf("Case %d - %s Begin\n", i, gCurCase->caseDesc); + printf("* Case %d - %s Begin *\n", i, gCurCase->caseDesc); if (gCurCase->fullCol) { gCurCase->bindColNum = gCurCase->colNum; @@ -5220,7 +5337,8 @@ void* runcase(TAOS *taos) { prepareCheckResult(taos); } - printf("Case %d End\n", i); + + printf("* Case %d - %s End *\n", i, gCurCase->caseDesc); } From c23260f94b70d16cee2e97d6c67a3fe751c538fc Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 20 Apr 2022 10:22:09 +0800 Subject: [PATCH 29/47] stmt --- tests/script/api/batchprepare.c | 1038 +------------------------------ 1 file changed, 32 insertions(+), 1006 deletions(-) diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index bdaa4f7172..5a11ec968f 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -63,31 +63,40 @@ typedef struct { } CaseCfg; CaseCfg gCase[] = { -// {"insert:MBSE1-FULL", tListLen(shortColList), shortColList, false, true, insertMBSETest1, 1, 10, 10, 0, 0, 10}, -// {"insert:MBSE1-FULL", tListLen(shortColList), shortColList, false, true, insertMBSETest1, 10, 100, 10, 0, 0, 10}, -// {"insert:MBSE1-FULL", tListLen(longColList), longColList, false, true, insertMBSETest1, 10, 10, 2, 0, 0, 1}, -// {"insert:MBSE1-C012", tListLen(longColList), longColList, false, false, insertMBSETest1, 10, 10, 2, 12, 0, 1}, -// {"insert:MBSE1-C002", tListLen(longColList), longColList, false, false, insertMBSETest1, 10, 10, 2, 2, 0, 1}, -// {"insert:MBSE2-FULL", tListLen(longColList), longColList, false, true, insertMBSETest2, 10, 10, 2, 0, 0, 1}, -// {"insert:MBSE2-C012", tListLen(longColList), longColList, false, false, insertMBSETest2, 10, 10, 2, 12, 0, 1}, -// {"insert:MBSE2-C002", tListLen(longColList), longColList, false, false, insertMBSETest2, 10, 10, 2, 2, 0, 1}, -// {"insert:MBME1-FULL", tListLen(longColList), longColList, false, true, insertMBMETest1, 10, 10, 2, 0, 0, 1}, -// {"insert:MBME1-C012", tListLen(longColList), longColList, false, false, insertMBMETest1, 10, 10, 2, 12, 0, 1}, -// {"insert:MBME1-C002", tListLen(longColList), longColList, false, false, insertMBMETest1, 10, 10, 2, 2, 0, 1}, -// {"insert:MBME2-FULL", tListLen(longColList), longColList, false, true, insertMBMETest2, 10, 10, 2, 0, 0, 1}, -// {"insert:MBME2-C012", tListLen(longColList), longColList, false, false, insertMBMETest2, 10, 10, 2, 12, 0, 1}, -// {"insert:MBME2-C002", tListLen(longColList), longColList, false, false, insertMBMETest2, 10, 10, 2, 2, 0, 1}, -// {"insert:MBME3-FULL", tListLen(longColList), longColList, false, true, insertMBMETest3, 10, 10, 2, 0, 0, 1}, -// {"insert:MBME3-C012", tListLen(longColList), longColList, false, false, insertMBMETest3, 10, 10, 2, 12, 0, 1}, -// {"insert:MBME3-C002", tListLen(longColList), longColList, false, false, insertMBMETest3, 10, 10, 2, 2, 0, 1}, -// {"insert:MBME4-FULL", tListLen(longColList), longColList, false, true, insertMBMETest4, 10, 10, 2, 0, 0, 1}, -// {"insert:MBME4-C012", tListLen(longColList), longColList, false, false, insertMBMETest4, 10, 10, 2, 12, 0, 1}, +#if 0 + {"insert:MBSE1-FULL", tListLen(shortColList), shortColList, false, true, insertMBSETest1, 1, 10, 10, 0, 0, 10}, + {"insert:MBSE1-FULL", tListLen(shortColList), shortColList, false, true, insertMBSETest1, 10, 100, 10, 0, 0, 10}, + + {"insert:MBSE1-FULL", tListLen(longColList), longColList, false, true, insertMBSETest1, 10, 10, 2, 0, 0, 1}, + {"insert:MBSE1-C012", tListLen(longColList), longColList, false, false, insertMBSETest1, 10, 10, 2, 12, 0, 1}, + {"insert:MBSE1-C002", tListLen(longColList), longColList, false, false, insertMBSETest1, 10, 10, 2, 2, 0, 1}, + + {"insert:MBSE2-FULL", tListLen(longColList), longColList, false, true, insertMBSETest2, 10, 10, 2, 0, 0, 1}, + {"insert:MBSE2-C012", tListLen(longColList), longColList, false, false, insertMBSETest2, 10, 10, 2, 12, 0, 1}, + {"insert:MBSE2-C002", tListLen(longColList), longColList, false, false, insertMBSETest2, 10, 10, 2, 2, 0, 1}, + + {"insert:MBME1-FULL", tListLen(longColList), longColList, false, true, insertMBMETest1, 10, 10, 2, 0, 0, 1}, + {"insert:MBME1-C012", tListLen(longColList), longColList, false, false, insertMBMETest1, 10, 10, 2, 12, 0, 1}, + {"insert:MBME1-C002", tListLen(longColList), longColList, false, false, insertMBMETest1, 10, 10, 2, 2, 0, 1}, + + {"insert:MBME2-FULL", tListLen(longColList), longColList, false, true, insertMBMETest2, 10, 10, 2, 0, 0, 1}, + {"insert:MBME2-C012", tListLen(longColList), longColList, false, false, insertMBMETest2, 10, 10, 2, 12, 0, 1}, + {"insert:MBME2-C002", tListLen(longColList), longColList, false, false, insertMBMETest2, 10, 10, 2, 2, 0, 1}, + + {"insert:MBME3-FULL", tListLen(longColList), longColList, false, true, insertMBMETest3, 10, 10, 2, 0, 0, 1}, + {"insert:MBME3-C012", tListLen(longColList), longColList, false, false, insertMBMETest3, 10, 10, 2, 12, 0, 1}, + {"insert:MBME3-C002", tListLen(longColList), longColList, false, false, insertMBMETest3, 10, 10, 2, 2, 0, 1}, +#endif + + {"insert:MBME4-FULL", tListLen(longColList), longColList, false, true, insertMBMETest4, 10, 10, 2, 0, 0, 1}, + {"insert:MBME4-C012", tListLen(longColList), longColList, false, false, insertMBMETest4, 10, 10, 2, 12, 0, 1}, {"insert:MBME4-C002", tListLen(longColList), longColList, false, false, insertMBMETest4, 10, 10, 2, 2, 0, 1}, }; CaseCfg *gCurCase = NULL; - +int32_t gTestNull = 1; +bool gTestAutoCreate = false; int32_t taosGetTimeOfDay(struct timeval *tv) { return gettimeofday(tv, NULL); @@ -704,510 +713,7 @@ int insertMBMETest4(TAOS_STMT *stmt) { return 0; } - -int stmt_scol_func1(TAOS_STMT *stmt) { - struct { - int64_t ts; - int8_t b; - int8_t v1; - int16_t v2; - int32_t v4; - int64_t v8; - float f4; - double f8; - char bin[40]; - char blob[80]; - } v = {0}; - int32_t len[10] = {sizeof(v.ts), sizeof(v.v1), sizeof(v.v2), sizeof(v.f4), sizeof(v.bin), sizeof(v.bin)}; - - TAOS_BIND_v2 params[10]; - params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[0].buffer_length = sizeof(v.ts); - params[0].buffer = &v.ts; - params[0].length = &len[0]; - params[0].is_null = NULL; - params[0].num = 1; - - params[1].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[1].buffer_length = sizeof(v.v1); - params[1].buffer = &v.v1; - params[1].length = &len[1]; - params[1].is_null = NULL; - params[1].num = 1; - - params[2].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[2].buffer_length = sizeof(v.v2); - params[2].buffer = &v.v2; - params[2].length = &len[2]; - params[2].is_null = NULL; - params[2].num = 1; - - params[3].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[3].buffer_length = sizeof(v.f4); - params[3].buffer = &v.f4; - params[3].length = &len[3]; - params[3].is_null = NULL; - params[3].num = 1; - - params[4].buffer_type = TSDB_DATA_TYPE_BINARY; - params[4].buffer_length = sizeof(v.bin); - params[4].buffer = v.bin; - params[4].length = &len[4]; - params[4].is_null = NULL; - params[4].num = 1; - - params[5].buffer_type = TSDB_DATA_TYPE_BINARY; - params[5].buffer_length = sizeof(v.bin); - params[5].buffer = v.bin; - params[5].length = &len[5]; - params[5].is_null = NULL; - params[5].num = 1; - - char *sql = "insert into ? (ts, v1,v2,f4,bin,bin2) values(?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); - } - - for (int zz = 0; zz < 10; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. error:%s\n", taos_stmt_errstr(stmt)); - exit(1); - } - v.ts = 1591060628000 + zz * 10; - for (int i = 0; i < 10; ++i) { - v.ts += 1; - - v.b = (int8_t)(i+zz*10) % 2; - v.v1 = (int8_t)(i+zz*10); - v.v2 = (int16_t)((i+zz*10) * 2); - v.v4 = (int32_t)((i+zz*10) * 4); - v.v8 = (int64_t)((i+zz*10) * 8); - v.f4 = (float)((i+zz*10) * 40); - v.f8 = (double)((i+zz*10) * 80); - for (int j = 0; j < sizeof(v.bin) - 1; ++j) { - v.bin[j] = (char)((i)%10 + '0'); - } - - taos_stmt_bind_param(stmt, params); - taos_stmt_add_batch(stmt); - } - } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement, error:%s.\n", taos_stmt_errstr(stmt)); - exit(1); - } - - return 0; -} - - -int stmt_scol_func2(TAOS_STMT *stmt) { - struct { - int64_t ts; - int8_t b; - int8_t v1; - int16_t v2; - int32_t v4; - int64_t v8; - float f4; - double f8; - char bin[40]; - char blob[80]; - } v = {0}; - - TAOS_BIND_v2 params[10]; - params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[0].buffer_length = sizeof(v.ts); - params[0].buffer = &v.ts; - params[0].length = ¶ms[0].buffer_length; - params[0].is_null = NULL; - params[0].num = 1; - - params[1].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[1].buffer_length = sizeof(v.v1); - params[1].buffer = &v.v1; - params[1].length = ¶ms[1].buffer_length; - params[1].is_null = NULL; - params[1].num = 1; - - params[2].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[2].buffer_length = sizeof(v.v2); - params[2].buffer = &v.v2; - params[2].length = ¶ms[2].buffer_length; - params[2].is_null = NULL; - params[2].num = 1; - - params[3].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[3].buffer_length = sizeof(v.f4); - params[3].buffer = &v.f4; - params[3].length = ¶ms[3].buffer_length; - params[3].is_null = NULL; - params[3].num = 1; - - params[4].buffer_type = TSDB_DATA_TYPE_BINARY; - params[4].buffer_length = sizeof(v.bin); - params[4].buffer = v.bin; - params[4].length = ¶ms[4].buffer_length; - params[4].is_null = NULL; - params[4].num = 1; - - params[5].buffer_type = TSDB_DATA_TYPE_BINARY; - params[5].buffer_length = sizeof(v.bin); - params[5].buffer = v.bin; - params[5].length = ¶ms[5].buffer_length; - params[5].is_null = NULL; - params[5].num = 1; - - char *sql = "insert into m0 (ts, v1,v2,f4,bin,bin2) values(?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - for (int zz = 0; zz < 10; zz++) { - v.ts = 1591060628000 + zz * 10; - for (int i = 0; i < 10; ++i) { - v.ts += 1; - - v.b = (int8_t)(i+zz*10) % 2; - v.v1 = (int8_t)(i+zz*10); - v.v2 = (int16_t)((i+zz*10) * 2); - v.v4 = (int32_t)((i+zz*10) * 4); - v.v8 = (int64_t)((i+zz*10) * 8); - v.f4 = (float)((i+zz*10) * 40); - v.f8 = (double)((i+zz*10) * 80); - for (int j = 0; j < sizeof(v.bin) - 1; ++j) { - v.bin[j] = (char)((i)%10 + '0'); - } - - taos_stmt_bind_param(stmt, params); - taos_stmt_add_batch(stmt); - } - } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - return 0; -} - - - - -//300 tables 60 records -int stmt_scol_func3(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[60]; - int8_t v1[60]; - int16_t v2[60]; - int32_t v4[60]; - int64_t v8[60]; - float f4[60]; - double f8[60]; - char bin[60][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - - int *lb = taosMemoryMalloc(60 * sizeof(int)); - - TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 900000*10); - char* is_null = taosMemoryMalloc(sizeof(char) * 60); - char* no_null = taosMemoryMalloc(sizeof(char) * 60); - - for (int i = 0; i < 60; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 9000000; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[10*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 10; - - params[i+1].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.v1; - params[i+1].length = NULL; - params[i+1].is_null = no_null; - params[i+1].num = 10; - - params[i+2].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+2].buffer_length = sizeof(int16_t); - params[i+2].buffer = v.v2; - params[i+2].length = NULL; - params[i+2].is_null = no_null; - params[i+2].num = 10; - - params[i+3].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+3].buffer_length = sizeof(float); - params[i+3].buffer = v.f4; - params[i+3].length = NULL; - params[i+3].is_null = no_null; - params[i+3].num = 10; - - params[i+4].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+4].buffer_length = 40; - params[i+4].buffer = v.bin; - params[i+4].length = lb; - params[i+4].is_null = no_null; - params[i+4].num = 10; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+5].buffer_length = 40; - params[i+5].buffer = v.bin; - params[i+5].length = lb; - params[i+5].is_null = no_null; - params[i+5].num = 10; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 54000000; ++i) { - v.ts[i] = tts + i; - } - - int64_t starttime = taosGetTimestampUs(); - - char *sql = "insert into ? (ts, v1,v2,f4,bin,bin2) values(?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - int id = 0; - for (int l = 0; l < 2; l++) { - for (int zz = 0; zz < 300; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - - taos_stmt_bind_param_batch(stmt, params + id * 10); - taos_stmt_add_batch(stmt); - } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - ++id; - } - - int64_t endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - - return 0; -} - - - -//10 tables 10 records single column bind -int stmt_scol_func4(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[60]; - int8_t v1[60]; - int16_t v2[60]; - int32_t v4[60]; - int64_t v8[60]; - float f4[60]; - double f8[60]; - char bin[60][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 1000 * 60); - - int *lb = taosMemoryMalloc(60 * sizeof(int)); - - TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 1000*10); - char* is_null = taosMemoryMalloc(sizeof(char) * 60); - char* no_null = taosMemoryMalloc(sizeof(char) * 60); - - for (int i = 0; i < 60; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 10000; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[10*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 2; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = no_null; - params[i+1].num = 2; - - params[i+2].buffer_type = TSDB_DATA_TYPE_INT; - params[i+2].buffer_length = sizeof(int32_t); - params[i+2].buffer = v.v4; - params[i+2].length = NULL; - params[i+2].is_null = no_null; - params[i+2].num = 2; - - params[i+3].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+3].buffer_length = sizeof(int64_t); - params[i+3].buffer = v.v8; - params[i+3].length = NULL; - params[i+3].is_null = no_null; - params[i+3].num = 2; - - params[i+4].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+4].buffer_length = sizeof(double); - params[i+4].buffer = v.f8; - params[i+4].length = NULL; - params[i+4].is_null = no_null; - params[i+4].num = 2; - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 60000; ++i) { - v.ts[i] = tts + i; - } - - int64_t starttime = taosGetTimestampUs(); - - char *sql = "insert into ? (ts,b,v4,v8,f8) values(?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - int id = 0; - for (int l = 0; l < 10; l++) { - for (int zz = 0; zz < 10; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - - for (int col=0; col < 10; ++col) { - taos_stmt_bind_single_param_batch(stmt, params + id++, col); - } - - taos_stmt_add_batch(stmt); - } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - } - - int64_t endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - - return 0; -} - - - - - -//1 tables 20 records -int stmt_scol_func5(TAOS_STMT *stmt) { - int32_t rowNum = gCurCase->rowNum; - int32_t bindRowNum = gCurCase->bindRowNum; - int32_t bindColNum = gCurCase->bindColNum; - int32_t nullNum = gCurCase->bindNullNum; - BindData data = {0}; - prepareData(&data); - - int64_t starttime = taosGetTimestampUs(); - - char *sql = "insert into ? (ts, v1,v2,f4,bin,bin2) values(?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - int id = 0; - for (int b = 0; b < (rowNum/bindRowNum); b++) { - //for (int b = 0; b < 2; b++) { - for (int zz = 0; zz < 10; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - - taos_stmt_bind_param_batch(stmt, data.pBind + b*bindColNum); - taos_stmt_add_batch(stmt); - } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - ++id; - } - - int64_t endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", rowNum, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - - destroyData(&data); - - return 0; -} - - #if 0 - - int stmt_func1(TAOS_STMT *stmt) { struct { int64_t ts; @@ -3423,488 +2929,6 @@ int stmt_funcb_autoctb_e5(TAOS_STMT *stmt) { return 0; } - - -//300 tables 60 records -int stmt_funcb1(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[60]; - int8_t v1[60]; - int16_t v2[60]; - int32_t v4[60]; - int64_t v8[60]; - float f4[60]; - double f8[60]; - char bin[60][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - - int *lb = taosMemoryMalloc(60 * sizeof(int)); - - TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 900000*10); - char* is_null = taosMemoryMalloc(sizeof(char) * 60); - char* no_null = taosMemoryMalloc(sizeof(char) * 60); - - for (int i = 0; i < 60; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 9000000; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[60*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 60; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 60; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 60; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 60; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 60; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 60; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 60; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 60; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 60; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 60; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 54000000; ++i) { - v.ts[i] = tts + i; - } - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - int id = 0; - for (int l = 0; l < 3000; l++) { - for (int zz = 0; zz < 300; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - - taos_stmt_bind_param_batch(stmt, params + id * 10); - taos_stmt_add_batch(stmt); - } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - ++id; - } - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - - return 0; -} - - -//1table 18000 reocrds -int stmt_funcb2(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[18000]; - int8_t v1[18000]; - int16_t v2[18000]; - int32_t v4[18000]; - int64_t v8[18000]; - float f4[18000]; - double f8[18000]; - char bin[18000][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - - int *lb = taosMemoryMalloc(18000 * sizeof(int)); - - TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 3000*10); - char* is_null = taosMemoryMalloc(sizeof(char) * 18000); - char* no_null = taosMemoryMalloc(sizeof(char) * 18000); - - for (int i = 0; i < 18000; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 30000; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[18000*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 18000; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 18000; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 18000; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 18000; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 18000; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 18000; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 18000; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 18000; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 18000; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 18000; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 54000000; ++i) { - v.ts[i] = tts + i; - } - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - int id = 0; - for (int l = 0; l < 10; l++) { - for (int zz = 0; zz < 300; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - - taos_stmt_bind_param_batch(stmt, params + id * 10); - taos_stmt_add_batch(stmt); - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - ++id; - - } - - } - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - - return 0; -} - - -//disorder -int stmt_funcb3(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[60]; - int8_t v1[60]; - int16_t v2[60]; - int32_t v4[60]; - int64_t v8[60]; - float f4[60]; - double f8[60]; - char bin[60][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - - int *lb = taosMemoryMalloc(60 * sizeof(int)); - - TAOS_BIND_v2 *params = taosMemoryCalloc(1, sizeof(TAOS_BIND_v2) * 900000*10); - char* is_null = taosMemoryMalloc(sizeof(char) * 60); - char* no_null = taosMemoryMalloc(sizeof(char) * 60); - - for (int i = 0; i < 60; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 9000000; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[60*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 60; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 60; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 60; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 60; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 60; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 60; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 60; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 60; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 60; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 60; - - } - - int64_t tts = 1591060628000; - int64_t ttt = 0; - for (int i = 0; i < 54000000; ++i) { - v.ts[i] = tts + i; - if (i > 0 && i%60 == 0) { - ttt = v.ts[i-1]; - v.ts[i-1] = v.ts[i-60]; - v.ts[i-60] = ttt; - } - } - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - int id = 0; - for (int l = 0; l < 3000; l++) { - for (int zz = 0; zz < 300; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - - taos_stmt_bind_param_batch(stmt, params + id * 10); - taos_stmt_add_batch(stmt); - } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - ++id; - } - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - - return 0; -} - - - - //samets int stmt_funcb4(TAOS_STMT *stmt) { struct { @@ -4936,7 +3960,6 @@ int stmt_funcb_sc3(TAOS_STMT *stmt) { } #endif - void prepareCheckResultImpl(TAOS *taos, char *tname, int printr, int expected) { char sql[255] = "SELECT * FROM "; TAOS_RES *result; @@ -5323,6 +4346,9 @@ void* runcase(TAOS *taos) { if (gCurCase->fullCol) { gCurCase->bindColNum = gCurCase->colNum; } + + gCurCase->bindNullNum = gTestNull; + gCurCase->autoCreate = gTestAutoCreate; for (int32_t n = 0; n < gCurCase->runTimes; ++n) { prepare(taos, gCurCase->colNum, gCurCase->colList, gCurCase->autoCreate); From c0bdae458405001bda11fdba379c23623d2e6f80 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 20 Apr 2022 11:10:01 +0800 Subject: [PATCH 30/47] stmt --- source/client/src/clientMain.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index d79c8ce227..63ed4ff20f 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -735,9 +735,6 @@ int taos_stmt_affected_rows(TAOS_STMT *stmt) { return stmtAffectedRows(stmt); } - - - int taos_stmt_close(TAOS_STMT *stmt) { if (stmt == NULL) { tscError("NULL parameter for %s", __FUNCTION__); From bb62a7d1ff281d0668d00e4053130c94bcc417f9 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 20 Apr 2022 11:26:20 +0800 Subject: [PATCH 31/47] stmt --- tests/script/api/batchprepare.c | 34 ++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 5a11ec968f..2bbb563d56 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -88,9 +88,9 @@ CaseCfg gCase[] = { {"insert:MBME3-C002", tListLen(longColList), longColList, false, false, insertMBMETest3, 10, 10, 2, 2, 0, 1}, #endif - {"insert:MBME4-FULL", tListLen(longColList), longColList, false, true, insertMBMETest4, 10, 10, 2, 0, 0, 1}, +// {"insert:MBME4-FULL", tListLen(longColList), longColList, false, true, insertMBMETest4, 10, 10, 2, 0, 0, 1}, {"insert:MBME4-C012", tListLen(longColList), longColList, false, false, insertMBMETest4, 10, 10, 2, 12, 0, 1}, - {"insert:MBME4-C002", tListLen(longColList), longColList, false, false, insertMBMETest4, 10, 10, 2, 2, 0, 1}, +// {"insert:MBME4-C002", tListLen(longColList), longColList, false, false, insertMBMETest4, 10, 10, 2, 2, 0, 1}, }; @@ -251,49 +251,49 @@ int32_t prepareColData(BindData *data, int32_t bindIdx, int32_t rowIdx, int32_t data->pBind[bindIdx].buffer_length = sizeof(bool); data->pBind[bindIdx].buffer = data->boolData + rowIdx; data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull; + data->pBind[bindIdx].is_null = data->isNull + rowIdx; break; case TSDB_DATA_TYPE_TINYINT: data->pBind[bindIdx].buffer_length = sizeof(int8_t); data->pBind[bindIdx].buffer = data->tinyData + rowIdx; data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull; + data->pBind[bindIdx].is_null = data->isNull + rowIdx; break; case TSDB_DATA_TYPE_SMALLINT: data->pBind[bindIdx].buffer_length = sizeof(int16_t); data->pBind[bindIdx].buffer = data->smallData + rowIdx; data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull; + data->pBind[bindIdx].is_null = data->isNull + rowIdx; break; case TSDB_DATA_TYPE_INT: data->pBind[bindIdx].buffer_length = sizeof(int32_t); data->pBind[bindIdx].buffer = data->intData + rowIdx; data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull; + data->pBind[bindIdx].is_null = data->isNull + rowIdx; break; case TSDB_DATA_TYPE_BIGINT: data->pBind[bindIdx].buffer_length = sizeof(int64_t); data->pBind[bindIdx].buffer = data->bigData + rowIdx; data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull; + data->pBind[bindIdx].is_null = data->isNull + rowIdx; break; case TSDB_DATA_TYPE_FLOAT: data->pBind[bindIdx].buffer_length = sizeof(float); data->pBind[bindIdx].buffer = data->floatData + rowIdx; data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull; + data->pBind[bindIdx].is_null = data->isNull + rowIdx; break; case TSDB_DATA_TYPE_DOUBLE: data->pBind[bindIdx].buffer_length = sizeof(double); data->pBind[bindIdx].buffer = data->doubleData + rowIdx; data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull; + data->pBind[bindIdx].is_null = data->isNull + rowIdx; break; case TSDB_DATA_TYPE_VARCHAR: data->pBind[bindIdx].buffer_length = gVarCharSize; data->pBind[bindIdx].buffer = data->binaryData + rowIdx * gVarCharSize; data->pBind[bindIdx].length = data->binaryLen; - data->pBind[bindIdx].is_null = data->isNull; + data->pBind[bindIdx].is_null = data->isNull + rowIdx; break; case TSDB_DATA_TYPE_TIMESTAMP: data->pBind[bindIdx].buffer_length = sizeof(int64_t); @@ -305,31 +305,31 @@ int32_t prepareColData(BindData *data, int32_t bindIdx, int32_t rowIdx, int32_t data->pBind[bindIdx].buffer_length = gVarCharSize; data->pBind[bindIdx].buffer = data->binaryData + rowIdx * gVarCharSize; data->pBind[bindIdx].length = data->binaryLen; - data->pBind[bindIdx].is_null = data->isNull; + data->pBind[bindIdx].is_null = data->isNull + rowIdx; break; case TSDB_DATA_TYPE_UTINYINT: data->pBind[bindIdx].buffer_length = sizeof(uint8_t); data->pBind[bindIdx].buffer = data->utinyData + rowIdx; data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull; + data->pBind[bindIdx].is_null = data->isNull + rowIdx; break; case TSDB_DATA_TYPE_USMALLINT: data->pBind[bindIdx].buffer_length = sizeof(uint16_t); data->pBind[bindIdx].buffer = data->usmallData + rowIdx; data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull; + data->pBind[bindIdx].is_null = data->isNull + rowIdx; break; case TSDB_DATA_TYPE_UINT: data->pBind[bindIdx].buffer_length = sizeof(uint32_t); data->pBind[bindIdx].buffer = data->uintData + rowIdx; data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull; + data->pBind[bindIdx].is_null = data->isNull + rowIdx; break; case TSDB_DATA_TYPE_UBIGINT: data->pBind[bindIdx].buffer_length = sizeof(uint64_t); data->pBind[bindIdx].buffer = data->ubigData + rowIdx; data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull; + data->pBind[bindIdx].is_null = data->isNull + rowIdx; break; default: printf("invalid col type:%d", dataType); @@ -3995,9 +3995,9 @@ void prepareCheckResultImpl(TAOS *taos, char *tname, int printr, int expecte } if (rows == expected) { - printf("%d rows are fetched as expectation\n", rows); + printf("%d rows are fetched as expected from %s\n", rows, tname); } else { - printf("!!!expect %d rows, but %d rows are fetched\n", expected, rows); + printf("!!!expect %d rows, but %d rows are fetched from %s\n", expected, rows, tname); exit(1); } From a016f1157a0abaa2001a7706bdb5ae3d8db9572f Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 20 Apr 2022 16:52:32 +0800 Subject: [PATCH 32/47] refactor: add unit test for json compare --- include/common/tdatablock.h | 32 +- source/libs/scalar/src/sclvector.c | 335 +++++++----------- .../libs/scalar/test/scalar/scalarTests.cpp | 235 ++++++++++-- 3 files changed, 358 insertions(+), 244 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index e40efd815d..1d6f496eea 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -57,11 +57,30 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet); #define colDataIsNull_var(pColumnInfoData, row) (pColumnInfoData->varmeta.offset[row] == -1) #define colDataSetNull_var(pColumnInfoData, row) (pColumnInfoData->varmeta.offset[row] = -1) +#define BitmapLen(_n) (((_n) + ((1 << NBIT) - 1)) >> NBIT) + +#define colDataGetVarData(p1_, r_) ((p1_)->pData + (p1_)->varmeta.offset[(r_)]) + +#define colDataGetNumData(p1_, r_) ((p1_)->pData + ((r_) * (p1_)->info.bytes)) +// SColumnInfoData, rowNumber +#define colDataGetData(p1_, r_) \ + ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? colDataGetVarData(p1_, r_) \ + : colDataGetNumData(p1_, r_)) + static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) { + if (pColumnInfoData->info.type == TSDB_DATA_TYPE_JSON){ + if(colDataIsNull_var(pColumnInfoData, row)){ + return true; + } + char *data = colDataGetVarData(pColumnInfoData, row); + return (*data == TSDB_DATA_TYPE_NULL); + } + if (!pColumnInfoData->hasNull) { return false; } - if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { + + if (pColumnInfoData->info.type== TSDB_DATA_TYPE_VARCHAR || pColumnInfoData->info.type == TSDB_DATA_TYPE_NCHAR) { return colDataIsNull_var(pColumnInfoData, row); } else { if (pColumnInfoData->nullbitmap == NULL) { @@ -99,16 +118,6 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u } } -#define BitmapLen(_n) (((_n) + ((1 << NBIT) - 1)) >> NBIT) - -#define colDataGetVarData(p1_, r_) ((p1_)->pData + (p1_)->varmeta.offset[(r_)]) - -#define colDataGetNumData(p1_, r_) ((p1_)->pData + ((r_) * (p1_)->info.bytes)) -// SColumnInfoData, rowNumber -#define colDataGetData(p1_, r_) \ - ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? colDataGetVarData(p1_, r_) \ - : colDataGetNumData(p1_, r_)) - static FORCE_INLINE void colDataAppendNULL(SColumnInfoData* pColumnInfoData, uint32_t currentRow) { // There is a placehold for each NULL value of binary or nchar type. if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { @@ -271,3 +280,4 @@ static FORCE_INLINE void blockCompressEncode(const SSDataBlock* pBlock, char* da #endif #endif /*_TD_COMMON_EP_H_*/ + diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index c2eafedf9b..ae8c395997 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -28,6 +28,75 @@ #define LEFT_COL ((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData)) #define RIGHT_COL ((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData)) +void convertNumberToNumber(const void *inData, void *outData, int8_t inType, int8_t outType){ + switch (outType) { + case TSDB_DATA_TYPE_BOOL: { + GET_TYPED_DATA(*((bool *)outData), bool, inType, inData); + break; + } + case TSDB_DATA_TYPE_TINYINT: { + GET_TYPED_DATA(*((int8_t *)outData), int8_t, inType, inData); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + GET_TYPED_DATA(*((int16_t *)outData), int16_t, inType, inData); + break; + } + case TSDB_DATA_TYPE_INT: { + GET_TYPED_DATA(*((int32_t *)outData), int32_t, inType, inData); + break; + } + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: { + GET_TYPED_DATA(*((int64_t *)outData), int64_t, inType, inData); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + GET_TYPED_DATA(*((uint8_t *)outData), uint8_t, inType, inData); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + GET_TYPED_DATA(*((uint16_t *)outData), uint16_t, inType, inData); + break; + } + case TSDB_DATA_TYPE_UINT: { + GET_TYPED_DATA(*((uint32_t *)outData), uint32_t, inType, inData); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + GET_TYPED_DATA(*((uint64_t *)outData), uint64_t, inType, inData); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + GET_TYPED_DATA(*((float *)outData), float, inType, inData); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + GET_TYPED_DATA(*((double *)outData), double, inType, inData); + break; + } + default:{ + ASSERT(0); + } + } +} + +void convertStringToDouble(const void *inData, void *outData, int8_t inType, int8_t outType){ + char *tmp = taosMemoryMalloc(varDataTLen(inData)); + int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(inData), varDataLen(inData), tmp); + if (len < 0) { + sclError("castConvert taosUcs4ToMbs error 1"); + } + + tmp[len] = 0; + + ASSERT(outType == TSDB_DATA_TYPE_DOUBLE); + double value = strtod(tmp, NULL); + + *((double *)outData) = value; + taosMemoryFreeClear(tmp); +} + typedef int64_t (*_getBigintValue_fn_t)(void *src, int32_t index); int64_t getVectorBigintValue_TINYINT(void *src, int32_t index) { @@ -64,6 +133,20 @@ int64_t getVectorBigintValue_BOOL(void *src, int32_t index) { return (int64_t)*((bool *)src + index); } +int64_t getVectorBigintValue_JSON(void *src, int32_t index){ + ASSERT(!colDataIsNull_var(((SColumnInfoData*)src), index)); + char *data = colDataGetVarData((SColumnInfoData*)src, index); + double out = 0; + if (*data == TSDB_DATA_TYPE_NULL){ + return 0; + } else if(*data == TSDB_DATA_TYPE_NCHAR) { // json inner type can not be BINARY + convertStringToDouble(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE); + } else { + convertNumberToNumber(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE); + } + return (int64_t)out; +} + _getBigintValue_fn_t getVectorBigintValueFn(int32_t srcType) { _getBigintValue_fn_t p = NULL; if(srcType==TSDB_DATA_TYPE_TINYINT) { @@ -90,6 +173,8 @@ _getBigintValue_fn_t getVectorBigintValueFn(int32_t srcType) { p = getVectorBigintValue_BIGINT; }else if(srcType==TSDB_DATA_TYPE_BOOL) { p = getVectorBigintValue_BOOL; + }else if(srcType==TSDB_DATA_TYPE_JSON) { + p = getVectorBigintValue_JSON; }else { assert(0); } @@ -200,75 +285,6 @@ static FORCE_INLINE void varToNchar(char* buf, SScalarParam* pOut, int32_t rowIn taosMemoryFree(t); } -void convertNumberToNumber(const void *inData, void *outData, int8_t inType, int8_t outType){ - switch (outType) { - case TSDB_DATA_TYPE_BOOL: { - GET_TYPED_DATA(*((bool *)outData), bool, inType, inData); - break; - } - case TSDB_DATA_TYPE_TINYINT: { - GET_TYPED_DATA(*((int8_t *)outData), int8_t, inType, inData); - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - GET_TYPED_DATA(*((int16_t *)outData), int16_t, inType, inData); - break; - } - case TSDB_DATA_TYPE_INT: { - GET_TYPED_DATA(*((int32_t *)outData), int32_t, inType, inData); - break; - } - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: { - GET_TYPED_DATA(*((int64_t *)outData), int64_t, inType, inData); - break; - } - case TSDB_DATA_TYPE_UTINYINT: { - GET_TYPED_DATA(*((uint8_t *)outData), uint8_t, inType, inData); - break; - } - case TSDB_DATA_TYPE_USMALLINT: { - GET_TYPED_DATA(*((uint16_t *)outData), uint16_t, inType, inData); - break; - } - case TSDB_DATA_TYPE_UINT: { - GET_TYPED_DATA(*((uint32_t *)outData), uint32_t, inType, inData); - break; - } - case TSDB_DATA_TYPE_UBIGINT: { - GET_TYPED_DATA(*((uint64_t *)outData), uint64_t, inType, inData); - break; - } - case TSDB_DATA_TYPE_FLOAT: { - GET_TYPED_DATA(*((float *)outData), float, inType, inData); - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - GET_TYPED_DATA(*((double *)outData), double, inType, inData); - break; - } - default:{ - ASSERT(0); - } - } -} - -void convertStringToDouble(const void *inData, void *outData, int8_t inType, int8_t outType){ - char *tmp = taosMemoryMalloc(varDataTLen(inData)); - int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(inData), varDataLen(inData), tmp); - if (len < 0) { - sclError("castConvert taosUcs4ToMbs error 1"); - } - - tmp[len] = 0; - - ASSERT(outType == TSDB_DATA_TYPE_DOUBLE); - double value = strtod(tmp, NULL); - - *((double *)outData) = value; - taosMemoryFreeClear(tmp); -} - //TODO opt performance, tmp is not needed. int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, int32_t inType, int32_t outType) { ASSERT(!IS_VAR_DATA_TYPE(outType)); @@ -345,12 +361,13 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in } double getVectorDoubleValue_JSON(void *src, int32_t index){ - ASSERT(!colDataIsNull_var(((SColumnInfoData*)src), index)); char *data = colDataGetVarData((SColumnInfoData*)src, index); double out = 0; - if(*data == TSDB_DATA_TYPE_NCHAR) { // json inner type can not be BINARY + if (*data == TSDB_DATA_TYPE_NULL){ + return out; + } else if(*data == TSDB_DATA_TYPE_NCHAR) { // json inner type can not be BINARY convertStringToDouble(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE); - } { + } else { convertNumberToNumber(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE); } return out; @@ -569,24 +586,24 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut) { } int8_t gConvertTypes[TSDB_DATA_TYPE_BLOB+1][TSDB_DATA_TYPE_BLOB+1] = { -/* NULL BOOL TINY SMAL INT BIG FLOA DOUB VARC TIME NCHA UTIN USMA UINT UBIG VARB JSON DECI BLOB */ +/* NULL BOOL TINY SMAL INT BIG FLOA DOUB VARC TIME NCHA UTIN USMA UINT UBIG JSON VARB DECI BLOB */ /*NULL*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -/*BOOL*/ 0, 0, 0, 3, 4, 5, 6, 7, 7, 9, 7, 0, 12, 13, 14, 7, 0, 0, 0, -/*TINY*/ 0, 0, 0, 3, 4, 5, 6, 7, 7, 9, 7, 3, 4, 5, 7, 7, 0, 0, 0, -/*SMAL*/ 0, 0, 0, 0, 4, 5, 6, 7, 7, 9, 7, 3, 4, 5, 7, 7, 0, 0, 0, -/*INT */ 0, 0, 0, 0, 0, 5, 6, 7, 7, 9, 7, 4, 4, 5, 7, 7, 0, 0, 0, -/*BIGI*/ 0, 0, 0, 0, 0, 0, 6, 7, 7, 0, 7, 5, 5, 5, 7, 7, 0, 0, 0, -/*FLOA*/ 0, 0, 0, 0, 0, 0, 0, 7, 7, 6, 7, 6, 6, 6, 6, 7, 0, 0, 0, -/*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 7, 0, 0, 0, +/*BOOL*/ 0, 0, 0, 3, 4, 5, 6, 7, 7, 9, 7, 0, 12, 13, 14, 0, 7, 0, 0, +/*TINY*/ 0, 0, 0, 3, 4, 5, 6, 7, 7, 9, 7, 3, 4, 5, 7, 0, 7, 0, 0, +/*SMAL*/ 0, 0, 0, 0, 4, 5, 6, 7, 7, 9, 7, 3, 4, 5, 7, 0, 7, 0, 0, +/*INT */ 0, 0, 0, 0, 0, 5, 6, 7, 7, 9, 7, 4, 4, 5, 7, 0, 7, 0, 0, +/*BIGI*/ 0, 0, 0, 0, 0, 0, 6, 7, 7, 0, 7, 5, 5, 5, 7, 0, 7, 0, 0, +/*FLOA*/ 0, 0, 0, 0, 0, 0, 0, 7, 7, 6, 7, 6, 6, 6, 6, 0, 7, 0, 0, +/*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 0, 7, 0, 0, /*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7, 7, 7, 7, 0, 0, 0, 0, -/*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 9, 9, 7, 7, 0, 0, 0, +/*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 9, 9, 7, 0, 7, 0, 0, /*NCHA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 0, 0, 0, 0, -/*UTIN*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 7, 0, 0, 0, -/*USMA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 7, 0, 0, 0, -/*UINT*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 7, 0, 0, 0, -/*UBIG*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, -/*VARB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*UTIN*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 0, 7, 0, 0, +/*USMA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 0, 7, 0, 0, +/*UINT*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 7, 0, 0, +/*UBIG*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, /*JSON*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*VARB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /*DECI*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /*BLOB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; @@ -677,7 +694,7 @@ enum { static int32_t doConvertHelper(SScalarParam* pDest, int32_t* convert, const SScalarParam* pParam, int32_t type) { SColumnInfoData* pCol = pParam->columnData; - if (IS_VAR_DATA_TYPE(pCol->info.type) || pCol->info.type != TSDB_DATA_TYPE_JSON) { + if (IS_VAR_DATA_TYPE(pCol->info.type) && pCol->info.type != TSDB_DATA_TYPE_JSON) { pDest->numOfRows = pParam->numOfRows; SDataType t = {.type = type, .bytes = tDataTypes[type].bytes}; @@ -718,10 +735,6 @@ static void vectorMathAddHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRig *output = getVectorDoubleValueFnLeft(LEFT_COL, i) + getVectorDoubleValueFnRight(RIGHT_COL, 0); } - pOutputCol->hasNull = pLeftCol->hasNull; - if (pOutputCol->hasNull) { - memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows)); - } } } @@ -735,12 +748,12 @@ static void vectorMathBigintAddHelper(SColumnInfoData* pLeftCol, SColumnInfoData colDataAppendNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { + if (colDataIsNull_s(pLeftCol, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore + } *output = getVectorBigintValueFnLeft(pLeftCol->pData, i) + getVectorBigintValueFnRight(pRightCol->pData, 0); } - pOutputCol->hasNull = pLeftCol->hasNull; - if (pOutputCol->hasNull) { - memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows)); - } } } @@ -771,8 +784,11 @@ char *getJsonValue(char *json, char *key){ //todo for (int i = 0; i < cols; ++i) { SColIdx *pColIdx = kvRowColIdxAt(json, i); char *data = kvRowColVal(json, pColIdx); - if(i == 0 && *data == TSDB_DATA_TYPE_NULL){ - return NULL; + if(i == 0){ + if(*data == TSDB_DATA_TYPE_NULL) { + return NULL; + } + continue; } if(memcmp(key, data, varDataTLen(data)) == 0){ return data + varDataTLen(data); @@ -798,7 +814,7 @@ void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO } char *pLeftData = colDataGetVarData(pLeft->columnData, i); char *value = getJsonValue(pLeftData, pRightData); - if (!value || *value == TSDB_DATA_TYPE_NULL) { + if (!value) { colDataSetNull_var(pOutputCol, i); pOutputCol->hasNull = true; continue; @@ -835,14 +851,6 @@ void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut } *output = getVectorBigintValueFnLeft(pLeftCol->pData, i) + getVectorBigintValueFnRight(pRightCol->pData, i); } - pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull); - if (pOutputCol->hasNull) { - int32_t numOfBitLen = BitmapLen(pLeft->numOfRows); - for (int32_t j = 0; j < numOfBitLen; ++j) { - pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j]; - } - } - } else if (pLeft->numOfRows == 1) { vectorMathBigintAddHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i); } else if (pRight->numOfRows == 1) { @@ -861,15 +869,6 @@ void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut } *output = getVectorDoubleValueFnLeft(LEFT_COL, i) + getVectorDoubleValueFnRight(RIGHT_COL, i); } - - pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull); - if (pOutputCol->hasNull) { - int32_t numOfBitLen = BitmapLen(pLeft->numOfRows); - for (int32_t j = 0; j < numOfBitLen; ++j) { - pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j]; - } - } - } else if (pLeft->numOfRows == 1) { vectorMathAddHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i); } else if (pRight->numOfRows == 1) { @@ -899,10 +898,6 @@ static void vectorMathSubHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRig *output = (getVectorDoubleValueFnLeft(LEFT_COL, i) - getVectorDoubleValueFnRight(RIGHT_COL, 0)) * factor; } - pOutputCol->hasNull = pLeftCol->hasNull; - if (pOutputCol->hasNull) { - memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows)); - } } } @@ -916,12 +911,12 @@ static void vectorMathBigintSubHelper(SColumnInfoData* pLeftCol, SColumnInfoData colDataAppendNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { + if (colDataIsNull_s(pLeftCol, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore + } *output = (getVectorBigintValueFnLeft(pLeftCol->pData, i) - getVectorBigintValueFnRight(pRightCol->pData, 0)) * factor; } - pOutputCol->hasNull = pLeftCol->hasNull; - if (pOutputCol->hasNull) { - memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows)); - } } } @@ -951,15 +946,6 @@ void vectorMathSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut } *output = getVectorBigintValueFnLeft(pLeftCol->pData, i) - getVectorBigintValueFnRight(pRightCol->pData, i); } - - pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull); - if (pOutputCol->hasNull) { - int32_t numOfBitLen = BitmapLen(pLeft->numOfRows); - for (int32_t j = 0; j < numOfBitLen; ++j) { - pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j]; - } - } - } else if (pLeft->numOfRows == 1) { vectorMathBigintSubHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, -1, i); } else if (pRight->numOfRows == 1) { @@ -978,15 +964,6 @@ void vectorMathSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut } *output = getVectorDoubleValueFnLeft(LEFT_COL, i) - getVectorDoubleValueFnRight(RIGHT_COL, i); } - - pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull); - if (pOutputCol->hasNull) { - int32_t numOfBitLen = BitmapLen(pLeft->numOfRows); - for (int32_t j = 0; j < numOfBitLen; ++j) { - pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j]; - } - } - } else if (pLeft->numOfRows == 1) { vectorMathSubHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, -1, i); } else if (pRight->numOfRows == 1) { @@ -1016,10 +993,6 @@ static void vectorMathMultiplyHelper(SColumnInfoData* pLeftCol, SColumnInfoData* *output = getVectorDoubleValueFnLeft(LEFT_COL, i) * getVectorDoubleValueFnRight(RIGHT_COL, 0); } - pOutputCol->hasNull = pLeftCol->hasNull; - if (pOutputCol->hasNull) { - memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows)); - } } } @@ -1047,15 +1020,6 @@ void vectorMathMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *output = getVectorDoubleValueFnLeft(LEFT_COL, i) * getVectorDoubleValueFnRight(RIGHT_COL, i); } - - pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull); - if (pOutputCol->hasNull) { - int32_t numOfBitLen = BitmapLen(pLeft->numOfRows); - for (int32_t j = 0; j < numOfBitLen; ++j) { - pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j]; - } - } - } else if (pLeft->numOfRows == 1) { vectorMathMultiplyHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i); } else if (pRight->numOfRows == 1) { @@ -1090,15 +1054,6 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p *output = getVectorDoubleValueFnLeft(LEFT_COL, i) /getVectorDoubleValueFnRight(RIGHT_COL, i); } - - pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull); - if (pOutputCol->hasNull) { - int32_t numOfBitLen = BitmapLen(pLeft->numOfRows); - for (int32_t j = 0; j < numOfBitLen; ++j) { - pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j]; - } - } - } else if (pLeft->numOfRows == 1) { if (colDataIsNull_s(pLeftCol, 0)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows); @@ -1111,10 +1066,6 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p *output = getVectorDoubleValueFnLeft(LEFT_COL, 0) / getVectorDoubleValueFnRight(RIGHT_COL, i); } - pOutputCol->hasNull = pRightCol->hasNull; - if (pOutputCol->hasNull) { - memcpy(pOutputCol->nullbitmap, pRightCol->nullbitmap, BitmapLen(pRight->numOfRows)); - } } } else if (pRight->numOfRows == 1) { if (colDataIsNull_s(pRightCol, 0)) { // Set pLeft->numOfRows NULL value @@ -1128,10 +1079,6 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p *output = getVectorDoubleValueFnLeft(LEFT_COL, i) / getVectorDoubleValueFnRight(RIGHT_COL, 0); } - pOutputCol->hasNull = pLeftCol->hasNull; - if (pOutputCol->hasNull) { - memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(pLeft->numOfRows)); - } } } @@ -1240,11 +1187,6 @@ void vectorMathMinus(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO *output = - getVectorDoubleValueFnLeft(LEFT_COL, i); } - pOutputCol->hasNull = pLeftCol->hasNull; - if (pOutputCol->hasNull) { - memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(pLeft->numOfRows)); - } - doReleaseVec(pLeftCol, leftConvert); } @@ -1315,10 +1257,6 @@ static void vectorBitAndHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRigh } *output = getVectorBigintValueFnLeft(LEFT_COL, i) & getVectorBigintValueFnRight(RIGHT_COL, 0); } - pOutputCol->hasNull = pLeftCol->hasNull; - if (pOutputCol->hasNull) { - memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows)); - } } } @@ -1345,15 +1283,6 @@ void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, } *output = getVectorBigintValueFnLeft(LEFT_COL, i) & getVectorBigintValueFnRight(RIGHT_COL, i); } - - pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull); - if (pOutputCol->hasNull) { - int32_t numOfBitLen = BitmapLen(pLeft->numOfRows); - for (int32_t j = 0; j < numOfBitLen; ++j) { - pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] & pRightCol->nullbitmap[j]; - } - } - } else if (pLeft->numOfRows == 1) { vectorBitAndHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i); } else if (pRight->numOfRows == 1) { @@ -1381,10 +1310,6 @@ static void vectorBitOrHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRight } *output = getVectorBigintValueFnLeft(LEFT_COL, i) | rx; } - pOutputCol->hasNull = pLeftCol->hasNull; - if (pOutputCol->hasNull) { - memcpy(pOutputCol->nullbitmap, pLeftCol->nullbitmap, BitmapLen(numOfRows)); - } } } @@ -1411,14 +1336,6 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, } *output = getVectorBigintValueFnLeft(LEFT_COL, i) | getVectorBigintValueFnRight(RIGHT_COL, i); } - - pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull); - if (pOutputCol->hasNull) { - int32_t numOfBitLen = BitmapLen(pLeft->numOfRows); - for (int32_t j = 0; j < numOfBitLen; ++j) { - pOutputCol->nullbitmap[j] = pLeftCol->nullbitmap[j] | pRightCol->nullbitmap[j]; - } - } } else if (pLeft->numOfRows == 1) { vectorBitOrHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i); } else if (pRight->numOfRows == 1) { @@ -1439,6 +1356,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * if (pRight->pHashFilter != NULL) { for (; i >= 0 && i < pLeft->numOfRows; i += step) { if (colDataIsNull_s(pLeft->columnData, i)) { + colDataAppendNULL(pOut->columnData, i); continue; } @@ -1452,6 +1370,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step) { if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { + colDataAppendNULL(pOut->columnData, i); continue; // TODO set null or ignore } @@ -1463,6 +1382,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * bool isJsonnull = false; convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull); if(isJsonnull){ + colDataAppendNULL(pOut->columnData, i); continue; // TODO set null or ignore } bool res = filterDoCompare(fp, optr, pLeftData, pRightData); @@ -1472,6 +1392,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * ASSERT(pLeft->pHashFilter == NULL); for (; i >= 0 && i < pLeft->numOfRows; i += step) { if (colDataIsNull_s(pLeft->columnData, i)) { + colDataAppendNULL(pOut->columnData, i); continue; } @@ -1482,6 +1403,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * bool isJsonnull = false; convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull); if(isJsonnull){ + colDataAppendNULL(pOut->columnData, i); continue; // TODO set null or ignore } bool res = filterDoCompare(fp, optr, pLeftData, pRightData); @@ -1490,6 +1412,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * } else if (pLeft->numOfRows == 1) { for (; i >= 0 && i < pRight->numOfRows; i += step) { if (colDataIsNull_s(pRight->columnData, i)) { + colDataAppendNULL(pOut->columnData, i); continue; } @@ -1500,6 +1423,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * bool isJsonnull = false; convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull); if(isJsonnull){ + colDataAppendNULL(pOut->columnData, i); continue; // TODO set null or ignore } bool res = filterDoCompare(fp, optr, pLeftData, pRightData); @@ -1589,10 +1513,6 @@ void vectorJsonContains(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { for(int32_t i = 0; i < pLeft->numOfRows; ++i) { int8_t v = colDataIsNull_s(pLeft->columnData, i)? 1:0; - if (v && pLeft->columnData->info.type == TSDB_DATA_TYPE_JSON){ - char *data = colDataGetVarData(pLeft->columnData, i); - v = (*data == TSDB_DATA_TYPE_NULL)? 1:0; - } colDataAppendInt8(pOut->columnData, i, &v); } pOut->numOfRows = pLeft->numOfRows; @@ -1601,10 +1521,6 @@ void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, void vectorNotNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { for(int32_t i = 0; i < pLeft->numOfRows; ++i) { int8_t v = colDataIsNull_s(pLeft->columnData, i)? 0:1; - if (v && pLeft->columnData->info.type == TSDB_DATA_TYPE_JSON){ - char *data = colDataGetVarData(pLeft->columnData, i); - v = (*data == TSDB_DATA_TYPE_NULL)? 0:1; - } colDataAppendInt8(pOut->columnData, i, &v); } pOut->numOfRows = pLeft->numOfRows; @@ -1671,3 +1587,4 @@ _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) { return NULL; } } + diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 8049e80590..502ab7edfe 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -38,6 +38,7 @@ #include "scalar.h" #include "nodes.h" #include "tlog.h" +#include "parUtil.h" #define _DEBUG_PRINT_ 0 @@ -911,17 +912,32 @@ TEST(constantTest, greater_and_lower) { } void makeJsonArrow(SSDataBlock **src, SNode **opNode, void *json, char *key){ - char key[32] = {0}; - memcpy(varDataVal(key), key, strlen(key)); - varDataLen(key) = strlen(key); + char keyVar[32] = {0}; + memcpy(varDataVal(keyVar), key, strlen(key)); + varDataLen(keyVar) = strlen(key); - SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; - scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, key); + SNode *pLeft = NULL, *pRight = NULL; + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, keyVar); scltMakeColumnNode(&pLeft, src, TSDB_DATA_TYPE_JSON, varDataLen(json), 1, json); scltMakeOpNode(opNode, OP_TYPE_JSON_GET_VALUE, TSDB_DATA_TYPE_JSON, pLeft, pRight); } -void makeOperator(SNode **opNode, SArray *blockList, int32_t resType, EOperatorType opType, int32_t rightType, void *rightData){ +void makeOperator(SNode **opNode, SArray *blockList, EOperatorType opType, int32_t rightType, void *rightData){ + int32_t resType = TSDB_DATA_TYPE_NULL; + if(opType == OP_TYPE_ADD || opType == OP_TYPE_SUB || opType == OP_TYPE_MULTI || + opType == OP_TYPE_DIV || opType == OP_TYPE_MOD || opType == OP_TYPE_MINUS){ + resType = TSDB_DATA_TYPE_DOUBLE; + }else if(opType == OP_TYPE_BIT_AND || opType == OP_TYPE_BIT_OR){ + resType = TSDB_DATA_TYPE_BIGINT; + }else if(opType == OP_TYPE_GREATER_THAN || opType == OP_TYPE_GREATER_EQUAL || + opType == OP_TYPE_LOWER_THAN || opType == OP_TYPE_LOWER_EQUAL || + opType == OP_TYPE_EQUAL || opType == OP_TYPE_NOT_EQUAL || + opType == OP_TYPE_IS_NULL || opType == OP_TYPE_IS_NOT_NULL || opType == OP_TYPE_IS_TRUE || + opType == OP_TYPE_LIKE || opType == OP_TYPE_NOT_LIKE || opType == OP_TYPE_MATCH || + opType == OP_TYPE_NMATCH){ + resType = TSDB_DATA_TYPE_BOOL; + } + SNode *right = NULL; scltMakeValueNode(&right, rightType, rightData); scltMakeOpNode(opNode, opType, resType, *opNode, right); @@ -932,15 +948,15 @@ void makeOperator(SNode **opNode, SArray *blockList, int32_t resType, EOperatorT scltMakeTargetNode(opNode, dataBlockId, slotId, *opNode); } -void makeCalculate(void *json, void *key, void *value, double exceptValue, EOperatorType opType){ +void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, double exceptValue, EOperatorType opType){ SArray *blockList = taosArrayInit(2, POINTER_BYTES); SSDataBlock *src = NULL; SNode *opNode = NULL; - makeJsonArrow(&src, &opNode, row, key); + makeJsonArrow(&src, &opNode, json, (char*)key); taosArrayPush(blockList, &src); - makeOperator(&opNode, blockList, TSDB_DATA_TYPE_DOUBLE, opType, TSDB_DATA_TYPE_INT, value); + makeOperator(&opNode, blockList, opType, rightType, rightData); int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); @@ -948,39 +964,209 @@ void makeCalculate(void *json, void *key, void *value, double exceptValue, EOper SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); ASSERT_EQ(res->info.rows, 1); SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); - ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_DOUBLE); - ASSERT_EQ(*((double *)colDataGetData(column, 0)), exceptValue); + + if(colDataIsNull_f(column->nullbitmap, 0)){ + ASSERT_EQ(DBL_MAX, exceptValue); + printf("result:NULL\n"); + + }else if(opType == OP_TYPE_ADD || opType == OP_TYPE_SUB || opType == OP_TYPE_MULTI || opType == OP_TYPE_DIV || + opType == OP_TYPE_MOD || opType == OP_TYPE_MINUS){ + double tmp = *((double *)colDataGetData(column, 0)); + ASSERT_TRUE(tmp == exceptValue); + printf("result:%lf\n", tmp); + }else if(opType == OP_TYPE_BIT_AND || opType == OP_TYPE_BIT_OR){ + ASSERT_EQ(*((int64_t *)colDataGetData(column, 0)), exceptValue); + printf("result:%ld\n", *((int64_t *)colDataGetData(column, 0))); + }else if(opType == OP_TYPE_GREATER_THAN || opType == OP_TYPE_GREATER_EQUAL || opType == OP_TYPE_LOWER_THAN || + opType == OP_TYPE_LOWER_EQUAL || opType == OP_TYPE_EQUAL || opType == OP_TYPE_NOT_EQUAL || + opType == OP_TYPE_IS_NULL || opType == OP_TYPE_IS_NOT_NULL || opType == OP_TYPE_IS_TRUE || + opType == OP_TYPE_LIKE || opType == OP_TYPE_NOT_LIKE || opType == OP_TYPE_MATCH || opType == OP_TYPE_NMATCH){ + ASSERT_EQ(*((bool *)colDataGetData(column, 0)), exceptValue); + printf("result:%d\n", *((bool *)colDataGetData(column, 0))); + } taosArrayDestroyEx(blockList, scltFreeDataBlock); nodesDestroyNode(opNode); } -TEST(columnTest, int_value_add_json_column) { +TEST(columnTest, json_column_arith_op) { scltInitLogFile(); char *rightv= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44}"; - char *key = "k1"; SKVRowBuilder kvRowBuilder; tdInitKVRowBuilder(&kvRowBuilder); parseJsontoTagData(rightv, &kvRowBuilder, NULL, 0); SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); - int32_t leftv = 1; - makeCalculate(row, key, &leftv, 5.0, OP_TYPE_ADD); + const int32_t len = 8; + EOperatorType op[len] = {OP_TYPE_ADD, OP_TYPE_SUB, OP_TYPE_MULTI, OP_TYPE_DIV, + OP_TYPE_MOD, OP_TYPE_MINUS, OP_TYPE_BIT_AND, OP_TYPE_BIT_OR}; + int32_t input[len] = {1, 8, 2, 2, 3, 0, -4, 9}; - leftv = 8; - makeCalculate(row, key, &leftv, -4, OP_TYPE_SUB); + printf("--------------------json int---------------------\n"); + char *key = "k1"; + double eRes[len] = {5.0, -4, 8.0, 2.0, 1.0, -4, 4&-4, 4|9}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes[i], op[i]); + } - leftv = 2; - makeCalculate(row, key, &leftv, 8.0, OP_TYPE_MULTI); + printf("--------------------json string---------------------\n"); - leftv = 2; - makeCalculate(row, key, &leftv, 2.0, OP_TYPE_DIV); + key = "k2"; + double eRes1[len] = {1.0, -8, 0, 0, 0, 0, 0, 9}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes1[i], op[i]); + } - leftv = 3; - makeCalculate(row, key, &leftv, 1.0, OP_TYPE_MODE); + printf("---------------------json null--------------------\n"); - makeCalculate(row, key, &leftv, -4, OP_TYPE_MINUS); + key = "k3"; + double eRes2[len] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes2[i], op[i]); + } + + printf("---------------------json bool--------------------\n"); + + key = "k4"; + double eRes3[len] = {2.0, -7, 2, 0.5, 1, -1, 1&-4, 1|9}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes3[i], op[i]); + } + + printf("----------------------json double-------------------\n"); + + key = "k5"; + double eRes4[len] = {6.44, -2.56, 10.88, 2.72, 2.44, -5.44, 5&-4, 5|9}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes4[i], op[i]); + } + + printf("---------------------json not exist--------------------\n"); + + key = "k10"; + double eRes5[len] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes5[i], op[i]); + } +} + +void *prepareNchar(char* rightData){ + int32_t len = 0; + int32_t inputLen = strlen(rightData); + + char* t = (char*)taosMemoryCalloc(1,(inputLen + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); + taosMbsToUcs4(rightData, inputLen, (TdUcs4*) varDataVal(t), inputLen * TSDB_NCHAR_SIZE, &len); + varDataSetLen(t, len); + return t; +} + +TEST(columnTest, json_column_logic_op) { + scltInitLogFile(); + char *rightv= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44,\"k6\":\"6.6hello\"}"; + + SKVRowBuilder kvRowBuilder; + tdInitKVRowBuilder(&kvRowBuilder); + parseJsontoTagData(rightv, &kvRowBuilder, NULL, 0); + SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); + + const int32_t len = 9; + const int32_t len1 = 4; + EOperatorType op[len+len1] = {OP_TYPE_GREATER_THAN, OP_TYPE_GREATER_EQUAL, OP_TYPE_LOWER_THAN, OP_TYPE_LOWER_EQUAL, OP_TYPE_EQUAL, OP_TYPE_NOT_EQUAL, + OP_TYPE_IS_NULL, OP_TYPE_IS_NOT_NULL, OP_TYPE_IS_TRUE, OP_TYPE_LIKE, OP_TYPE_NOT_LIKE, OP_TYPE_MATCH, OP_TYPE_NMATCH}; + + int32_t input[len] = {1, 8, 2, 2, 3, 0, 0, 0, 0}; + char *inputNchar[len1] = {"hell_", "hel%", "hell", "llll"}; + + printf("--------------------json int---------------------\n"); + char *key = "k1"; + bool eRes[len+len1] = {true, false, false, false, false, true, false, true, true, false, false, false, false}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes[i], op[i]); + } + for(int i = len; i < len + len1; i++){ + void* rightData = prepareNchar(inputNchar[i-len]); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes[i], op[i]); + taosMemoryFree(rightData); + } + + printf("--------------------json string---------------------\n"); + + key = "k2"; + bool eRes1[len+len1] = {false, false, true, true, false, false, false, true, false, true, false, true, true}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes1[i], op[i]); + } + for(int i = len; i < len + len1; i++){ + void* rightData = prepareNchar(inputNchar[i-len]); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes1[i], op[i]); + taosMemoryFree(rightData); + } + + printf("--------------------json null---------------------\n"); + + key = "k3"; + double eRes2[len+len1] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, true, false, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes2[i], op[i]); + } + for(int i = len; i < len + len1; i++){ + void* rightData = prepareNchar(inputNchar[i-len]); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes2[i], op[i]); + taosMemoryFree(rightData); + } + + printf("--------------------json bool---------------------\n"); + + key = "k4"; + bool eRes3[len+len1] = {false, false, true, true, false, true, false, true, true, false, false, false, false}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes3[i], op[i]); + } + for(int i = len; i < len + len1; i++){ + void* rightData = prepareNchar(inputNchar[i-len]); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes3[i], op[i]); + taosMemoryFree(rightData); + } + + printf("--------------------json double---------------------\n"); + + key = "k5"; + bool eRes4[len+len1] = {true, false, false, false, false, true, false, true, true, false, false, false, false}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes4[i], op[i]); + } + for(int i = len; i < len + len1; i++){ + void* rightData = prepareNchar(inputNchar[i-len]); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes4[i], op[i]); + taosMemoryFree(rightData); + } + + printf("--------------------json double---------------------\n"); + + key = "k6"; + bool eRes5[len+len1] = {true, false, false, false, false, true, false, true, true, false, false, false, true}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes5[i], op[i]); + } + for(int i = len; i < len + len1; i++){ + void* rightData = prepareNchar(inputNchar[i-len]); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes5[i], op[i]); + taosMemoryFree(rightData); + } + + printf("---------------------json not exist--------------------\n"); + + key = "k10"; + double eRes10[len+len1] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, true, false, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes10[i], op[i]); + } + for(int i = len; i < len + len1; i++){ + void* rightData = prepareNchar(inputNchar[i-len]); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes10[i], op[i]); + taosMemoryFree(rightData); + } } TEST(columnTest, smallint_value_add_int_column) { @@ -3171,3 +3357,4 @@ int main(int argc, char** argv) { } #pragma GCC diagnostic pop + From 177b1c96a5a61f58e9b9c104e1f7aac91a774305 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 20 Apr 2022 17:46:29 +0800 Subject: [PATCH 33/47] udfd final program ongoing --- include/util/tlog.h | 1 + source/common/src/tglobal.c | 2 + source/libs/function/inc/fnLog.h | 24 + source/libs/function/inc/tudf.h | 3 + source/libs/function/src/udfd.c | 761 +++++++++++++++++++------------ source/util/src/tlog.c | 2 + 6 files changed, 490 insertions(+), 303 deletions(-) create mode 100644 source/libs/function/inc/fnLog.h diff --git a/include/util/tlog.h b/include/util/tlog.h index d3ab9b0bfb..32421a59cc 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -59,6 +59,7 @@ extern int32_t sDebugFlag; extern int32_t tsdbDebugFlag; extern int32_t tqDebugFlag; extern int32_t fsDebugFlag; +extern int32_t fnDebugFlag; int32_t taosInitLog(const char *logName, int32_t maxFiles); void taosCloseLog(); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 49f4afc12b..a8cba255db 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -284,6 +284,7 @@ static int32_t taosAddServerLogCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, 0) != 0) return -1; if (cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, 0) != 0) return -1; if (cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "fnDebugFlag", fnDebugFlag, 0, 255, 0) != 0) return -1; return 0; } @@ -464,6 +465,7 @@ static void taosSetServerLogCfg(SConfig *pCfg) { tsdbDebugFlag = cfgGetItem(pCfg, "tsdbDebugFlag")->i32; tqDebugFlag = cfgGetItem(pCfg, "tqDebugFlag")->i32; fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32; + fnDebugFlag = cfgGetItem(pCfg, "fnDebugFlag")->i32; } static int32_t taosSetClientCfg(SConfig *pCfg) { diff --git a/source/libs/function/inc/fnLog.h b/source/libs/function/inc/fnLog.h new file mode 100644 index 0000000000..f572948907 --- /dev/null +++ b/source/libs/function/inc/fnLog.h @@ -0,0 +1,24 @@ +// +// Created by slzhou on 22-4-20. +// + +#ifndef TDENGINE_FNLOG_H +#define TDENGINE_FNLOG_H +#include "tlog.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define fnFatal(...) { if (fnDebugFlag & DEBUG_FATAL) { taosPrintLog("FN FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} +#define fnError(...) { if (fnDebugFlag & DEBUG_ERROR) { taosPrintLog("FN ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} +#define fnWarn(...) { if (fnDebugFlag & DEBUG_WARN) { taosPrintLog("FN WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} +#define fnInfo(...) { if (fnDebugFlag & DEBUG_INFO) { taosPrintLog("FN ", DEBUG_INFO, 255, __VA_ARGS__); }} +#define fnDebug(...) { if (fnDebugFlag & DEBUG_DEBUG) { taosPrintLog("FN ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }} +#define fnTrace(...) { if (fnDebugFlag & DEBUG_TRACE) { taosPrintLog("FN ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }} + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_FNLOG_H diff --git a/source/libs/function/inc/tudf.h b/source/libs/function/inc/tudf.h index 8ec02c777f..d7bb6dfb46 100644 --- a/source/libs/function/inc/tudf.h +++ b/source/libs/function/inc/tudf.h @@ -26,6 +26,9 @@ extern "C" { #endif +#define UDF_LISTEN_PIPE_NAME_LEN 32 +#define UDF_LISTEN_PIPE_NAME_PREFIX "udf.sock." + //====================================================================================== //begin API to taosd and qworker diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index a02c94c109..641b8535c3 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -12,10 +12,10 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ - #include "uv.h" #include "os.h" -#include "tlog.h" +#include "fnLog.h" +#include "thash.h" #include "tudf.h" #include "tudfInt.h" @@ -25,336 +25,381 @@ #include "tmsg.h" #include "trpc.h" -static uv_loop_t *loop; +typedef struct SUdfdContext { + uv_loop_t *loop; + char listenPipeName[UDF_LISTEN_PIPE_NAME_LEN]; + void *clientRpc; + + uv_mutex_t udfsMutex; + SHashObj* udfsHash; + + bool printVersion; +} SUdfdContext; + +SUdfdContext global; typedef struct SUdfdUvConn { - uv_stream_t *client; - char *inputBuf; - int32_t inputLen; - int32_t inputCap; - int32_t inputTotal; + uv_stream_t *client; + char *inputBuf; + int32_t inputLen; + int32_t inputCap; + int32_t inputTotal; } SUdfdUvConn; typedef struct SUvUdfWork { - uv_stream_t *client; - uv_buf_t input; - uv_buf_t output; + uv_stream_t *client; + uv_buf_t input; + uv_buf_t output; } SUvUdfWork; +typedef enum { + UDF_STATE_INIT = 0, + UDF_STATE_LOADING, + UDF_STATE_READY, + UDF_STATE_UNLOADING +} EUdfState; + typedef struct SUdf { - int32_t refCount; + int32_t refCount; + EUdfState state; + uv_mutex_t lock; + uv_cond_t condReady; - char name[16]; - int8_t type; + char name[16]; + int8_t type; + char path[PATH_MAX]; - uv_lib_t lib; - TUdfScalarProcFunc scalarProcFunc; - TUdfFreeUdfColumnFunc freeUdfColumn; + uv_lib_t lib; + TUdfScalarProcFunc scalarProcFunc; + TUdfFreeUdfColumnFunc freeUdfColumn; } SUdf; -//TODO: low priority: change name onxxx to xxxCb, and udfc or udfd as prefix -//TODO: add private udf structure. +// TODO: low priority: change name onxxx to xxxCb, and udfc or udfd as prefix +// TODO: add private udf structure. typedef struct SUdfHandle { - SUdf *udf; + SUdf *udf; } SUdfHandle; +int32_t udfdLoadUdf(char* udfName, SUdf* udf) { + strcpy(udf->name, udfName); -void udfdProcessRequest(uv_work_t *req) { - SUvUdfWork *uvUdf = (SUvUdfWork *) (req->data); - SUdfRequest request = {0}; - decodeUdfRequest(uvUdf->input.base, &request); - - switch (request.type) { - case UDF_TASK_SETUP: { - debugPrint("%s", "process setup request"); - SUdf *udf = taosMemoryMalloc(sizeof(SUdf)); - udf->refCount = 0; - SUdfSetupRequest *setup = &request.setup; - strcpy(udf->name, setup->udfName); - //TODO: retrive udf info from mnode - char* path = "libudf1.so"; - int err = uv_dlopen(path, &udf->lib); - if (err != 0) { - debugPrint("can not load library %s. error: %s", path, uv_strerror(err)); - //TODO set error - } - - char normalFuncName[TSDB_FUNC_NAME_LEN] = {0}; - strcpy(normalFuncName, setup->udfName); - //TODO error, multi-thread, same udf, lock it - //TODO find all functions normal, init, destroy, normal, merge, finalize - uv_dlsym(&udf->lib, normalFuncName, (void **) (&udf->scalarProcFunc)); - char freeFuncName[TSDB_FUNC_NAME_LEN + 6] = {0}; - char *freeSuffix = "_free"; - strncpy(freeFuncName, normalFuncName, strlen(normalFuncName)); - strncat(freeFuncName, freeSuffix, strlen(freeSuffix)); - uv_dlsym(&udf->lib, freeFuncName, (void **)(&udf->freeUdfColumn)); - - SUdfHandle *handle = taosMemoryMalloc(sizeof(SUdfHandle)); - handle->udf = udf; - udf->refCount++; - //TODO: allocate private structure and call init function and set it to handle - SUdfResponse rsp; - rsp.seqNum = request.seqNum; - rsp.type = request.type; - rsp.code = 0; - rsp.setupRsp.udfHandle = (int64_t) (handle); - int32_t len = encodeUdfResponse(NULL, &rsp); - rsp.msgLen = len; - void *bufBegin = taosMemoryMalloc(len); - void *buf = bufBegin; - encodeUdfResponse(&buf, &rsp); - - uvUdf->output = uv_buf_init(bufBegin, len); - - taosMemoryFree(uvUdf->input.base); - break; - } - - case UDF_TASK_CALL: { - debugPrint("%s", "process call request"); - SUdfCallRequest *call = &request.call; - SUdfHandle *handle = (SUdfHandle *) (call->udfHandle); - SUdf *udf = handle->udf; - - SUdfDataBlock input = {0}; - convertDataBlockToUdfDataBlock(&call->block, &input); - SUdfColumn output = {0}; - //TODO: call different functions according to call type, for now just calar - if (call->callType == TSDB_UDF_CALL_SCALA_PROC) { - udf->scalarProcFunc(input, &output); - } - - SUdfResponse response = {0}; - SUdfResponse *rsp = &response; - if (call->callType == TSDB_UDF_CALL_SCALA_PROC) { - rsp->seqNum = request.seqNum; - rsp->type = request.type; - rsp->code = 0; - SUdfCallResponse *subRsp = &rsp->callRsp; - subRsp->callType = call->callType; - convertUdfColumnToDataBlock(&output, &subRsp->resultData); - } - - int32_t len = encodeUdfResponse(NULL, rsp); - rsp->msgLen = len; - void *bufBegin = taosMemoryMalloc(len); - void *buf = bufBegin; - encodeUdfResponse(&buf, rsp); - uvUdf->output = uv_buf_init(bufBegin, len); - - //TODO: free - udf->freeUdfColumn(&output); - - taosMemoryFree(uvUdf->input.base); - break; - } - case UDF_TASK_TEARDOWN: { - debugPrint("%s", "process teardown request"); - - SUdfTeardownRequest *teardown = &request.teardown; - SUdfHandle *handle = (SUdfHandle *) (teardown->udfHandle); - SUdf *udf = handle->udf; - udf->refCount--; - if (udf->refCount == 0) { - uv_dlclose(&udf->lib); - taosMemoryFree(udf); - } - //TODO: call destroy and free udf private - taosMemoryFree(handle); - - SUdfResponse response; - SUdfResponse *rsp = &response; - rsp->seqNum = request.seqNum; - rsp->type = request.type; - rsp->code = 0; - int32_t len = encodeUdfResponse(NULL, rsp); - rsp->msgLen = len; - void *bufBegin = taosMemoryMalloc(len); - void *buf = bufBegin; - encodeUdfResponse(&buf, rsp); - uvUdf->output = uv_buf_init(bufBegin, len); - - taosMemoryFree(uvUdf->input.base); - break; - } - default: { - break; - } - + // TODO: retrive udf info from mnode + char *path = "libudf1.so"; + int err = uv_dlopen(path, &udf->lib); + if (err != 0) { + fnError("can not load library %s. error: %s", path, uv_strerror(err)); + // TODO set error } + char normalFuncName[TSDB_FUNC_NAME_LEN] = {0}; + strcpy(normalFuncName, udfName); + // TODO error, multi-thread, same udf, lock it + // TODO find all functions normal, init, destroy, normal, merge, finalize + uv_dlsym(&udf->lib, normalFuncName, (void **)(&udf->scalarProcFunc)); + char freeFuncName[TSDB_FUNC_NAME_LEN + 6] = {0}; + char *freeSuffix = "_free"; + strncpy(freeFuncName, normalFuncName, strlen(normalFuncName)); + strncat(freeFuncName, freeSuffix, strlen(freeSuffix)); + uv_dlsym(&udf->lib, freeFuncName, (void **)(&udf->freeUdfColumn)); + return 0; +} + +void udfdProcessRequest(uv_work_t *req) { + SUvUdfWork *uvUdf = (SUvUdfWork *)(req->data); + SUdfRequest request = {0}; + decodeUdfRequest(uvUdf->input.base, &request); + + switch (request.type) { + case UDF_TASK_SETUP: { + //TODO: tracable id from client. connect, setup, call, teardown + fnInfo("%"PRId64" setup request. udf name: %s", request.seqNum, request.setup.udfName); + SUdfSetupRequest *setup = &request.setup; + + SUdf* udf = NULL; + uv_mutex_lock(&global.udfsMutex); + SUdf** udfInHash = taosHashGet(global.udfsHash, request.setup.udfName, TSDB_FUNC_NAME_LEN); + if (*udfInHash) { + ++(*udfInHash)->refCount; + udf = *udfInHash; + uv_mutex_unlock(&global.udfsMutex); + } else { + SUdf *udfNew = taosMemoryCalloc(1, sizeof(SUdf)); + udfNew->refCount = 1; + udfNew->state = UDF_STATE_INIT; + + uv_mutex_init(&udfNew->lock); + uv_cond_init(&udfNew->condReady); + udf = udfNew; + taosHashPut(global.udfsHash, request.setup.udfName, TSDB_FUNC_NAME_LEN, &udfNew, sizeof(&udfNew)); + uv_mutex_unlock(&global.udfsMutex); + } + + uv_mutex_lock(&udf->lock); + if (udf->state == UDF_STATE_INIT) { + udf->state = UDF_STATE_LOADING; + udfdLoadUdf(setup->udfName, udf); + udf->state = UDF_STATE_READY; + uv_cond_broadcast(&udf->condReady); + uv_mutex_unlock(&udf->lock); + } else { + while (udf->state != UDF_STATE_READY) { + uv_cond_wait(&udf->condReady, &udf->lock); + } + uv_mutex_unlock(&udf->lock); + } + SUdfHandle *handle = taosMemoryMalloc(sizeof(SUdfHandle)); + handle->udf = udf; + // TODO: allocate private structure and call init function and set it to handle + SUdfResponse rsp; + rsp.seqNum = request.seqNum; + rsp.type = request.type; + rsp.code = 0; + rsp.setupRsp.udfHandle = (int64_t)(handle); + int32_t len = encodeUdfResponse(NULL, &rsp); + rsp.msgLen = len; + void *bufBegin = taosMemoryMalloc(len); + void *buf = bufBegin; + encodeUdfResponse(&buf, &rsp); + + uvUdf->output = uv_buf_init(bufBegin, len); + + taosMemoryFree(uvUdf->input.base); + break; + } + + case UDF_TASK_CALL: { + SUdfCallRequest *call = &request.call; + fnDebug("%"PRId64 "call request. call type %d, handle: %"PRIx64, request.seqNum, call->callType, call->udfHandle); + SUdfHandle *handle = (SUdfHandle *)(call->udfHandle); + SUdf *udf = handle->udf; + + SUdfDataBlock input = {0}; + convertDataBlockToUdfDataBlock(&call->block, &input); + SUdfColumn output = {0}; + // TODO: call different functions according to call type, for now just calar + if (call->callType == TSDB_UDF_CALL_SCALA_PROC) { + udf->scalarProcFunc(input, &output); + } + + SUdfResponse response = {0}; + SUdfResponse *rsp = &response; + if (call->callType == TSDB_UDF_CALL_SCALA_PROC) { + rsp->seqNum = request.seqNum; + rsp->type = request.type; + rsp->code = 0; + SUdfCallResponse *subRsp = &rsp->callRsp; + subRsp->callType = call->callType; + convertUdfColumnToDataBlock(&output, &subRsp->resultData); + } + + int32_t len = encodeUdfResponse(NULL, rsp); + rsp->msgLen = len; + void *bufBegin = taosMemoryMalloc(len); + void *buf = bufBegin; + encodeUdfResponse(&buf, rsp); + uvUdf->output = uv_buf_init(bufBegin, len); + + // TODO: free udf column + udf->freeUdfColumn(&output); + + taosMemoryFree(uvUdf->input.base); + break; + } + case UDF_TASK_TEARDOWN: { + SUdfTeardownRequest *teardown = &request.teardown; + fnInfo("teardown. %"PRId64"handle:%"PRIx64, request.seqNum, teardown->udfHandle) + SUdfHandle *handle = (SUdfHandle *)(teardown->udfHandle); + SUdf *udf = handle->udf; + bool unloadUdf = false; + uv_mutex_lock(&global.udfsMutex); + udf->refCount--; + if (udf->refCount == 0) { + unloadUdf = true; + taosHashRemove(global.udfsHash, udf->name, TSDB_FUNC_NAME_LEN); + } + uv_mutex_unlock(&global.udfsMutex); + if (unloadUdf) { + uv_cond_destroy(&udf->condReady); + uv_mutex_destroy(&udf->lock); + uv_dlclose(&udf->lib); + taosMemoryFree(udf); + } + // TODO: call destroy and free udf private + taosMemoryFree(handle); + + SUdfResponse response; + SUdfResponse *rsp = &response; + rsp->seqNum = request.seqNum; + rsp->type = request.type; + rsp->code = 0; + int32_t len = encodeUdfResponse(NULL, rsp); + rsp->msgLen = len; + void *bufBegin = taosMemoryMalloc(len); + void *buf = bufBegin; + encodeUdfResponse(&buf, rsp); + uvUdf->output = uv_buf_init(bufBegin, len); + + taosMemoryFree(uvUdf->input.base); + break; + } + default: { + break; + } + } } void udfdOnWrite(uv_write_t *req, int status) { - debugPrint("%s", "server after writing to pipe"); - if (status < 0) { - debugPrint("Write error %s", uv_err_name(status)); - } - SUvUdfWork *work = (SUvUdfWork *) req->data; - debugPrint("\tlength: %zu", work->output.len); - taosMemoryFree(work->output.base); - taosMemoryFree(work); - taosMemoryFree(req); + SUvUdfWork *work = (SUvUdfWork *)req->data; + if (status < 0) { + //TODO:log error and process it. + } + fnDebug("send response. length:%zu, status: %s", work->output.len, uv_err_name(status)); + taosMemoryFree(work->output.base); + taosMemoryFree(work); + taosMemoryFree(req); } - void udfdSendResponse(uv_work_t *work, int status) { - debugPrint("%s", "send response"); - SUvUdfWork *udfWork = (SUvUdfWork *) (work->data); + SUvUdfWork *udfWork = (SUvUdfWork *)(work->data); - uv_write_t *write_req = taosMemoryMalloc(sizeof(uv_write_t)); - write_req->data = udfWork; - uv_write(write_req, udfWork->client, &udfWork->output, 1, udfdOnWrite); + uv_write_t *write_req = taosMemoryMalloc(sizeof(uv_write_t)); + write_req->data = udfWork; + uv_write(write_req, udfWork->client, &udfWork->output, 1, udfdOnWrite); - taosMemoryFree(work); + taosMemoryFree(work); } void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) { - debugPrint("%s", "server allocate buffer for read"); - SUdfdUvConn *ctx = handle->data; - int32_t msgHeadSize = sizeof(int32_t) + sizeof(int64_t); - if (ctx->inputCap == 0) { - ctx->inputBuf = taosMemoryMalloc(msgHeadSize); - if (ctx->inputBuf) { - ctx->inputLen = 0; - ctx->inputCap = msgHeadSize; - ctx->inputTotal = -1; + SUdfdUvConn *ctx = handle->data; + int32_t msgHeadSize = sizeof(int32_t) + sizeof(int64_t); + if (ctx->inputCap == 0) { + ctx->inputBuf = taosMemoryMalloc(msgHeadSize); + if (ctx->inputBuf) { + ctx->inputLen = 0; + ctx->inputCap = msgHeadSize; + ctx->inputTotal = -1; - buf->base = ctx->inputBuf; - buf->len = ctx->inputCap; - } else { - //TODO: log error - buf->base = NULL; - buf->len = 0; - } + buf->base = ctx->inputBuf; + buf->len = ctx->inputCap; } else { - ctx->inputCap = ctx->inputTotal > ctx->inputCap ? ctx->inputTotal : ctx->inputCap; - void *inputBuf = taosMemoryRealloc(ctx->inputBuf, ctx->inputCap); - if (inputBuf) { - ctx->inputBuf = inputBuf; - buf->base = ctx->inputBuf + ctx->inputLen; - buf->len = ctx->inputCap - ctx->inputLen; - } else { - //TODO: log error - buf->base = NULL; - buf->len = 0; - } + // TODO: log error + buf->base = NULL; + buf->len = 0; } - debugPrint("\tinput buf cap - len - total : %d - %d - %d", ctx->inputCap, ctx->inputLen, ctx->inputTotal); - + } else { + ctx->inputCap = ctx->inputTotal > ctx->inputCap ? ctx->inputTotal : ctx->inputCap; + void *inputBuf = taosMemoryRealloc(ctx->inputBuf, ctx->inputCap); + if (inputBuf) { + ctx->inputBuf = inputBuf; + buf->base = ctx->inputBuf + ctx->inputLen; + buf->len = ctx->inputCap - ctx->inputLen; + } else { + // TODO: log error + buf->base = NULL; + buf->len = 0; + } + } + fnDebug("allocate buf. input buf cap - len - total : %d - %d - %d", ctx->inputCap, ctx->inputLen, ctx->inputTotal); } bool isUdfdUvMsgComplete(SUdfdUvConn *pipe) { - if (pipe->inputTotal == -1 && pipe->inputLen >= sizeof(int32_t)) { - pipe->inputTotal = *(int32_t *) (pipe->inputBuf); - } - if (pipe->inputLen == pipe->inputCap && pipe->inputTotal == pipe->inputCap) { - return true; - } - return false; + if (pipe->inputTotal == -1 && pipe->inputLen >= sizeof(int32_t)) { + pipe->inputTotal = *(int32_t *)(pipe->inputBuf); + } + if (pipe->inputLen == pipe->inputCap && pipe->inputTotal == pipe->inputCap) { + fnDebug("receive request complete. length %d", pipe->inputLen); + return true; + } + return false; } void udfdHandleRequest(SUdfdUvConn *conn) { - uv_work_t *work = taosMemoryMalloc(sizeof(uv_work_t)); - SUvUdfWork *udfWork = taosMemoryMalloc(sizeof(SUvUdfWork)); - udfWork->client = conn->client; - udfWork->input = uv_buf_init(conn->inputBuf, conn->inputLen); - conn->inputBuf = NULL; - conn->inputLen = 0; - conn->inputCap = 0; - conn->inputTotal = -1; - work->data = udfWork; - uv_queue_work(loop, work, udfdProcessRequest, udfdSendResponse); + uv_work_t *work = taosMemoryMalloc(sizeof(uv_work_t)); + SUvUdfWork *udfWork = taosMemoryMalloc(sizeof(SUvUdfWork)); + udfWork->client = conn->client; + udfWork->input = uv_buf_init(conn->inputBuf, conn->inputLen); + conn->inputBuf = NULL; + conn->inputLen = 0; + conn->inputCap = 0; + conn->inputTotal = -1; + work->data = udfWork; + uv_queue_work(global.loop, work, udfdProcessRequest, udfdSendResponse); } void udfdPipeCloseCb(uv_handle_t *pipe) { - SUdfdUvConn *conn = pipe->data; - taosMemoryFree(conn->client); - taosMemoryFree(conn->inputBuf); - taosMemoryFree(conn); + SUdfdUvConn *conn = pipe->data; + taosMemoryFree(conn->client); + taosMemoryFree(conn->inputBuf); + taosMemoryFree(conn); } -void udfdUvHandleError(SUdfdUvConn *conn) { - uv_close((uv_handle_t *) conn->client, udfdPipeCloseCb); -} +void udfdUvHandleError(SUdfdUvConn *conn) { uv_close((uv_handle_t *)conn->client, udfdPipeCloseCb); } void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { - debugPrint("%s, nread: %zd", "read from pipe", nread); + fnDebug("udf read %zu bytes from client", nread); + if (nread == 0) return; - if (nread == 0) return; + SUdfdUvConn *conn = client->data; - SUdfdUvConn *conn = client->data; - - if (nread > 0) { - conn->inputLen += nread; - if (isUdfdUvMsgComplete(conn)) { - udfdHandleRequest(conn); - } else { - //log error or continue; - } - return; + if (nread > 0) { + conn->inputLen += nread; + if (isUdfdUvMsgComplete(conn)) { + udfdHandleRequest(conn); + } else { + // log error or continue; } + return; + } - if (nread < 0) { - debugPrint("Read error %s", uv_err_name(nread)); - if (nread == UV_EOF) { - //TODO check more when close - } else { - } - udfdUvHandleError(conn); + if (nread < 0) { + fnDebug("Receive error %s", uv_err_name(nread)); + if (nread == UV_EOF) { + // TODO check more when close + } else { } + udfdUvHandleError(conn); + } } void udfdOnNewConnection(uv_stream_t *server, int status) { - debugPrint("%s", "on new connection"); - if (status < 0) { - // TODO - return; - } + fnDebug("new connection"); + if (status < 0) { + // TODO + return; + } - uv_pipe_t *client = (uv_pipe_t *) taosMemoryMalloc(sizeof(uv_pipe_t)); - uv_pipe_init(loop, client, 0); - if (uv_accept(server, (uv_stream_t *) client) == 0) { - SUdfdUvConn *ctx = taosMemoryMalloc(sizeof(SUdfdUvConn)); - ctx->client = (uv_stream_t *) client; - ctx->inputBuf = 0; - ctx->inputLen = 0; - ctx->inputCap = 0; - client->data = ctx; - ctx->client = (uv_stream_t *) client; - uv_read_start((uv_stream_t *) client, udfdAllocBuffer, udfdPipeRead); - } else { - uv_close((uv_handle_t *) client, NULL); - } + uv_pipe_t *client = (uv_pipe_t *)taosMemoryMalloc(sizeof(uv_pipe_t)); + uv_pipe_init(global.loop, client, 0); + if (uv_accept(server, (uv_stream_t *)client) == 0) { + SUdfdUvConn *ctx = taosMemoryMalloc(sizeof(SUdfdUvConn)); + ctx->client = (uv_stream_t *)client; + ctx->inputBuf = 0; + ctx->inputLen = 0; + ctx->inputCap = 0; + client->data = ctx; + ctx->client = (uv_stream_t *)client; + uv_read_start((uv_stream_t *)client, udfdAllocBuffer, udfdPipeRead); + } else { + uv_close((uv_handle_t *)client, NULL); + } } void removeListeningPipe(int sig) { - uv_fs_t req; - uv_fs_unlink(loop, &req, "udf.sock", NULL); - exit(0); + uv_fs_t req; + uv_fs_unlink(global.loop, &req, "udf.sock", NULL); + exit(0); } -typedef struct SServerContext { - void *clientRpc; -} SUdfdContext; +void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { return; } - -void udfdProcessRpcRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { - - return; -} - -int32_t fetchUdfFuncInfo(void *clientRpc, SEpSet* pEpSet, char* udfNames[], int32_t numOfUdfs) { +int32_t udfdFillUdfInfoFromMNode(void *clientRpc, SEpSet *pEpSet, char* udfName, SUdf* udf) { SRetrieveFuncReq retrieveReq = {0}; retrieveReq.numOfFuncs = 1; retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN); - for (int32_t i = 0; i < numOfUdfs; ++i) { - taosArrayPush(retrieveReq.pFuncNames, udfNames[i]); - } + taosArrayPush(retrieveReq.pFuncNames, udfName); int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); - void* pReq = rpcMallocCont(contLen); + void *pReq = rpcMallocCont(contLen); tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq); taosArrayDestroy(retrieveReq.pFuncNames); @@ -368,66 +413,176 @@ int32_t fetchUdfFuncInfo(void *clientRpc, SEpSet* pEpSet, char* udfNames[], int3 SRetrieveFuncRsp retrieveRsp = {0}; tDeserializeSRetrieveFuncRsp(rpcRsp.pCont, rpcRsp.contLen, &retrieveRsp); - SFuncInfo* pFuncInfo = (SFuncInfo*)taosArrayGet(retrieveRsp.pFuncInfos, 0); + SFuncInfo *pFuncInfo = (SFuncInfo *)taosArrayGet(retrieveRsp.pFuncInfos, 0); + char path[PATH_MAX] = {0}; + taosGetTmpfilePath("/tmp", "libudf", path); + TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); + // TODO check for failure of flush to disk + taosWriteFile(file, pFuncInfo->pCode, pFuncInfo->codeSize); + taosCloseFile(&file); + strncpy(udf->path, path, strlen(path)); taosArrayDestroy(retrieveRsp.pFuncInfos); rpcFreeCont(rpcRsp.pCont); return 0; } -int32_t openUdfdClientRpc(SUdfdContext *ctx) { +int32_t udfdOpenClientRpc() { char *pass = "taosdata"; char *user = "root"; - char secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0}; - taosEncryptPass_c((uint8_t*)pass, strlen(pass), secretEncrypt); + char secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0}; + taosEncryptPass_c((uint8_t *)pass, strlen(pass), secretEncrypt); SRpcInit rpcInit = {0}; - rpcInit.label = (char*)"UDFD"; + rpcInit.label = (char *)"UDFD"; rpcInit.numOfThreads = 1; rpcInit.cfp = udfdProcessRpcRsp; rpcInit.sessions = 1024; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.idleTime = 30 * 1000; - rpcInit.parent = ctx; + rpcInit.parent = &global; - rpcInit.user = (char*)user; - rpcInit.ckey = (char*)"key"; - rpcInit.secret = (char*)secretEncrypt; + rpcInit.user = (char *)user; + rpcInit.ckey = (char *)"key"; + rpcInit.secret = (char *)secretEncrypt; rpcInit.spi = 1; - ctx->clientRpc = rpcOpen(&rpcInit); + global.clientRpc = rpcOpen(&rpcInit); return 0; } -int32_t closeUdfdClientRpc(SUdfdContext *ctx) { - rpcClose(ctx->clientRpc); +int32_t udfdCloseClientRpc() { + rpcClose(global.clientRpc); + return 0; +} + +static void udfdPrintVersion() { +#ifdef TD_ENTERPRISE + char *releaseName = "enterprise"; +#else + char *releaseName = "community"; +#endif + printf("%s version: %s compatible_version: %s\n", releaseName, version, compatible_version); + printf("gitinfo: %s\n", gitinfo); + printf("buildInfo: %s\n", buildinfo); +} + +static int32_t udfdParseArgs(int32_t argc, char *argv[]) { + for (int32_t i = 1; i < argc; ++i) { + if (strcmp(argv[i], "-c") == 0) { + if (i < argc - 1) { + if (strlen(argv[++i]) >= PATH_MAX) { + printf("config file path overflow"); + return -1; + } + tstrncpy(configDir, argv[i], PATH_MAX); + } else { + printf("'-c' requires a parameter, default is %s\n", configDir); + return -1; + } + } else if (strcmp(argv[i], "-V") == 0) { + global.printVersion = true; + } else { + } + } return 0; } -int main() { - debugPrint("libuv version: %x", UV_VERSION_HEX); - - loop = uv_default_loop(); - uv_fs_t req; - uv_fs_unlink(loop, &req, "udf.sock", NULL); - - uv_pipe_t server; - uv_pipe_init(loop, &server, 0); - - signal(SIGINT, removeListeningPipe); - - int r; - if ((r = uv_pipe_bind(&server, "udf.sock"))) { - debugPrint("Bind error %s\n", uv_err_name(r)); - removeListeningPipe(0); - return 1; - } - if ((r = uv_listen((uv_stream_t *) &server, 128, udfdOnNewConnection))) { - debugPrint("Listen error %s", uv_err_name(r)); - return 2; - } - uv_run(loop, UV_RUN_DEFAULT); - uv_loop_close(loop); +static int32_t udfdInitLog() { + char logName[12] = {0}; + snprintf(logName, sizeof(logName), "%slog", "udfd"); + return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, 0); +} + +static int32_t udfdUvInit() { + uv_loop_t* loop = taosMemoryMalloc(sizeof(uv_loop_t)); + if (loop) { + uv_loop_init(loop); + } + global.loop = loop; + char dnodeId[8] = {0}; + size_t dnodeIdSize; + uv_os_getenv("DNODE_ID", dnodeId, &dnodeIdSize); + char listenPipeName[32] = {0}; + snprintf(listenPipeName, sizeof(listenPipeName), "%s%s", UDF_LISTEN_PIPE_NAME_PREFIX, dnodeId); + strcpy(global.listenPipeName, listenPipeName); + + uv_fs_t req; + uv_fs_unlink(global.loop, &req, global.listenPipeName, NULL); + + uv_pipe_t server; + uv_pipe_init(global.loop, &server, 0); + + signal(SIGINT, removeListeningPipe); + + int r; + fnInfo("bind to pipe %s", global.listenPipeName); + if ((r = uv_pipe_bind(&server, listenPipeName))) { + fnError("Bind error %s", uv_err_name(r)); + removeListeningPipe(0); + return -1; + } + if ((r = uv_listen((uv_stream_t *)&server, 128, udfdOnNewConnection))) { + fnError("Listen error %s", uv_err_name(r)); + removeListeningPipe(0); + return -2; + } + return 0; +} + +static int32_t udfdRun() { + global.udfsHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + uv_mutex_init(&global.udfsMutex); + + //TOOD: client rpc to fetch udf function info from mnode + if (udfdOpenClientRpc() != 0) { + fnError("open rpc connection to mnode failure"); + return -1; + } + + if (udfdUvInit() != 0) { + fnError("uv init failure"); + return -2; + } + + fnInfo("start the udfd"); + int code = uv_run(global.loop, UV_RUN_DEFAULT); + fnInfo("udfd stopped. result: %s", uv_err_name(code)); + int codeClose = uv_loop_close(global.loop); + fnDebug("uv loop close. result: %s", uv_err_name(codeClose)); + udfdCloseClientRpc(); + uv_mutex_destroy(&global.udfsMutex); + taosHashCleanup(global.udfsHash); + return code; +} + +int main(int argc, char* argv[]) { + if (!taosCheckSystemIsSmallEnd()) { + printf("failed to start since on non-small-end machines\n"); + return -1; + } + + if (udfdParseArgs(argc, argv) != 0) { + printf("failed to start since parse args error\n"); + return -1; + } + + if (global.printVersion) { + udfdPrintVersion(); + return 0; + } + + if (udfdInitLog() != 0) { + printf("failed to start since init log error\n"); + return -1; + } + + if (taosInitCfg(configDir, NULL, NULL, NULL, 0) != 0) { + fnError("failed to start since read config error"); + return -1; + } + + return udfdRun(); } diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 3dce260b10..6da4913114 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -91,6 +91,7 @@ int32_t sDebugFlag = 135; int32_t tsdbDebugFlag = 131; int32_t tqDebugFlag = 135; int32_t fsDebugFlag = 135; +int32_t fnDebugFlag = 135; int64_t dbgEmptyW = 0; int64_t dbgWN = 0; @@ -752,6 +753,7 @@ void taosSetAllDebugFlag(int32_t flag) { tsdbDebugFlag = flag; tqDebugFlag = flag; fsDebugFlag = flag; + fnDebugFlag = flag; uInfo("all debug flag are set to %d", flag); } From b77731702e025bc864d50437c8390f5f993ad50b Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 20 Apr 2022 18:27:00 +0800 Subject: [PATCH 34/47] stmt --- include/common/trow.h | 37 ++++ include/libs/parser/parser.h | 4 +- source/client/inc/clientStmt.h | 3 + source/client/src/clientMain.c | 4 +- source/client/src/clientStmt.c | 53 ++++- source/libs/parser/src/parInsert.c | 117 +++++++++-- source/libs/parser/src/parser.c | 6 +- tests/script/api/batchprepare.c | 304 +++++++++++++++++++++++++---- 8 files changed, 467 insertions(+), 61 deletions(-) diff --git a/include/common/trow.h b/include/common/trow.h index 40462a7eef..83bcc67bb5 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -684,6 +684,43 @@ static int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { return TSDB_CODE_SUCCESS; } +/** + * @brief The invoker is responsible for memory alloc/dealloc. + * + * @param pBuilder + * @param pBuf Output buffer of STSRow + */ +static int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf) { + pBuilder->pBuf = (STSRow *)pBuf; + if (!pBuilder->pBuf) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + + TASSERT(pBuilder->nBitmaps > 0 && pBuilder->flen > 0); + + uint32_t len = 0; + switch (pBuilder->rowType) { + case TD_ROW_TP: +#ifdef TD_SUPPORT_BITMAP + pBuilder->pBitmap = tdGetBitmapAddrTp(pBuilder->pBuf, pBuilder->flen); +#endif + break; + case TD_ROW_KV: +#ifdef TD_SUPPORT_BITMAP + pBuilder->pBitmap = tdGetBitmapAddrKv(pBuilder->pBuf, pBuilder->nBoundCols); +#endif + break; + default: + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + return TSDB_CODE_SUCCESS; +} + + /** * @brief 由调用方管理存储空间的分配及释放，一次输入多个参数 * diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index a7377655c7..58482735ba 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -76,6 +76,7 @@ typedef struct SQuery { } SQuery; int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery); +bool isInsertSql(const char* pStr, size_t length); void qDestroyQuery(SQuery* pQueryNode); @@ -87,7 +88,8 @@ int32_t qCloneStmtDataBlock(void** pDst, void* pSrc); void qFreeStmtDataBlock(void* pDataBlock); int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc); void qDestroyStmtDataBlock(void* pBlock); -int32_t qBindStmtColsValue(void *pDataBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen); +int32_t qBindStmtColsValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen); +int32_t qBindStmtSingleColValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen, int32_t colIdx, int32_t rowNum); int32_t qBuildStmtColFields(void *pDataBlock, int32_t *fieldNum, TAOS_FIELD** fields); int32_t qBuildStmtTagFields(void *pBlock, void *boundTags, int32_t *fieldNum, TAOS_FIELD** fields); int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, SName *pName, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen); diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index 2b9fe4005f..219257ba74 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -51,6 +51,8 @@ typedef struct SStmtBindInfo { bool needParse; uint64_t tbUid; uint64_t tbSuid; + int32_t sBindRowNum; + int32_t sBindLastIdx; int8_t tbType; void* boundTags; char* tbName; @@ -77,6 +79,7 @@ typedef struct SStmtSQLInfo { typedef struct STscStmt { STscObj* taos; SCatalog* pCatalog; + int32_t affectedRows; SStmtSQLInfo sql; SStmtExecInfo exec; diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 63ed4ff20f..1e6f7c4fa1 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -662,13 +662,13 @@ int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int c return terrno; } - if (colIdx <= 0) { + if (colIdx < 0) { tscError("invalid bind column idx %d", colIdx); terrno = TSDB_CODE_INVALID_PARA; return terrno; } - return stmtBindBatch(stmt, bind, colIdx); /* TODO */ + return stmtBindBatch(stmt, bind, colIdx); } int taos_stmt_add_batch(TAOS_STMT *stmt) { diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 987bf4fa4b..e6a47714d9 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -185,7 +185,7 @@ int32_t stmtCleanSQLInfo(STscStmt* pStmt) { void *pIter = taosHashIterate(pStmt->sql.pTableCache, NULL); while (pIter) { - SStmtTableCache* pCache = *(SStmtTableCache**)pIter; + SStmtTableCache* pCache = (SStmtTableCache*)pIter; qDestroyStmtDataBlock(pCache->pDataBlock); destroyBoundColumnInfo(pCache->boundTags); @@ -265,6 +265,20 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } +int32_t stmtResetStmt(STscStmt* pStmt) { + STMT_ERR_RET(stmtCleanSQLInfo(pStmt)); + + pStmt->sql.pTableCache = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + if (NULL == pStmt->sql.pTableCache) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + STMT_ERR_RET(terrno); + } + + pStmt->sql.status = STMT_INIT; + + return TSDB_CODE_SUCCESS; +} + TAOS_STMT *stmtInit(TAOS *taos) { STscObj* pObj = (STscObj*)taos; @@ -294,7 +308,7 @@ int stmtPrepare(TAOS_STMT *stmt, const char *sql, unsigned long length) { STscStmt* pStmt = (STscStmt*)stmt; if (pStmt->sql.status >= STMT_PREPARE) { - STMT_ERR_RET(stmtCleanSQLInfo(pStmt)); + STMT_ERR_RET(stmtResetStmt(pStmt)); } STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_PREPARE)); @@ -424,8 +438,23 @@ int stmtBindBatch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int32_t colIdx) { tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid); STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - - qBindStmtColsValue(*pDataBlock, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen); + + if (colIdx < 0) { + qBindStmtColsValue(*pDataBlock, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen); + } else { + if (colIdx != (pStmt->bInfo.sBindLastIdx + 1) && colIdx != 0) { + tscError("bind column index not in sequence"); + STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + pStmt->bInfo.sBindLastIdx = colIdx; + + if (0 == colIdx) { + pStmt->bInfo.sBindRowNum = bind->num; + } + + qBindStmtSingleColValue(*pDataBlock, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen, colIdx, pStmt->bInfo.sBindRowNum); + } return TSDB_CODE_SUCCESS; } @@ -453,6 +482,8 @@ int stmtExec(TAOS_STMT *stmt) { STMT_ERR_JRET(pStmt->exec.pRequest->code); + pStmt->affectedRows += taos_affected_rows(pStmt->exec.pRequest); + _return: stmtCleanExecInfo(pStmt, (code ? false : true)); @@ -464,7 +495,9 @@ _return: int stmtClose(TAOS_STMT *stmt) { - return TSDB_CODE_SUCCESS; + STscStmt* pStmt = (STscStmt*)stmt; + + STMT_RET(stmtCleanSQLInfo(pStmt)); } const char *stmtErrstr(TAOS_STMT *stmt) { @@ -482,18 +515,24 @@ const char *stmtErrstr(TAOS_STMT *stmt) { } int stmtAffectedRows(TAOS_STMT *stmt) { - return TSDB_CODE_SUCCESS; + return ((STscStmt*)stmt)->affectedRows; } int stmtIsInsert(TAOS_STMT *stmt, int *insert) { STscStmt* pStmt = (STscStmt*)stmt; - *insert = (STMT_TYPE_INSERT == pStmt->sql.type || STMT_TYPE_MULTI_INSERT == pStmt->sql.type); + if (pStmt->sql.type) { + *insert = (STMT_TYPE_INSERT == pStmt->sql.type || STMT_TYPE_MULTI_INSERT == pStmt->sql.type); + } else { + *insert = isInsertSql(pStmt->sql.sqlStr, 0); + } return TSDB_CODE_SUCCESS; } int stmtGetParamNum(TAOS_STMT *stmt, int *nums) { + STMT_ERR_RET(stmtFetchColFields(stmt, nums, NULL)); + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index b6d835846c..2a38552049 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1318,7 +1318,8 @@ int32_t qBindStmtColsValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32 SRowBuilder* pBuilder = &pDataBlock->rowBuilder; SMemParam param = {.rb = pBuilder}; SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; - + int32_t rowNum = bind->num; + CHECK_CODE(initRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo)); CHECK_CODE(allocateMemForSize(pDataBlock, extendedRowSize * bind->num)); @@ -1330,6 +1331,14 @@ int32_t qBindStmtColsValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32 // 1. set the parsed value from sql string for (int c = 0; c < spd->numOfBound; ++c) { SSchema* pColSchema = &pSchema[spd->boundColumns[c] - 1]; + + if (bind[c].buffer_type != pColSchema->type) { + return buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type"); + } + + if (bind[c].num != rowNum) { + return buildInvalidOperationMsg(&pBuf, "row number in each bind param should be the same"); + } param.schema = pColSchema; getSTSRowAppendInfo(pBuilder->rowType, spd, c, ¶m.toffset, ¶m.colIdx); @@ -1367,7 +1376,7 @@ int32_t qBindStmtColsValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32 pDataBlock->size += extendedRowSize; } - + SSubmitBlk *pBlocks = (SSubmitBlk *)(pDataBlock->pData); if (TSDB_CODE_SUCCESS != setBlockInfo(pBlocks, pDataBlock, bind->num)) { return buildInvalidOperationMsg(&pBuf, "too many rows in sql, total number of rows should be less than 32767"); @@ -1376,18 +1385,100 @@ int32_t qBindStmtColsValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32 return TSDB_CODE_SUCCESS; } +int32_t qBindStmtSingleColValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen, int32_t colIdx, int32_t rowNum) { + STableDataBlocks *pDataBlock = (STableDataBlocks *)pBlock; + SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); + int32_t extendedRowSize = getExtendedRowSize(pDataBlock); + SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; + SRowBuilder* pBuilder = &pDataBlock->rowBuilder; + SMemParam param = {.rb = pBuilder}; + SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; + bool rowStart = (0 == colIdx); + bool rowEnd = ((colIdx + 1) == spd->numOfBound); -int32_t buildBoundFields(SParsedDataColInfo *boundInfo, SSchema *pSchema, int32_t *fieldNum, TAOS_FIELD** fields) { - *fields = taosMemoryCalloc(boundInfo->numOfBound, sizeof(TAOS_FIELD)); - if (NULL == *fields) { - return TSDB_CODE_OUT_OF_MEMORY; + if (rowStart) { + CHECK_CODE(initRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo)); + CHECK_CODE(allocateMemForSize(pDataBlock, extendedRowSize * bind->num)); + } + + for (int32_t r = 0; r < bind->num; ++r) { + STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size + extendedRowSize * r); // skip the SSubmitBlk header + if (rowStart) { + tdSRowResetBuf(pBuilder, row); + } else { + tdSRowGetBuf(pBuilder, row); + } + + SSchema* pColSchema = &pSchema[spd->boundColumns[colIdx] - 1]; + + if (bind->buffer_type != pColSchema->type) { + return buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type"); + } + + if (bind->num != rowNum) { + return buildInvalidOperationMsg(&pBuf, "row number in each bind param should be the same"); + } + + param.schema = pColSchema; + getSTSRowAppendInfo(pBuilder->rowType, spd, colIdx, ¶m.toffset, ¶m.colIdx); + + if (bind->is_null && bind->is_null[r]) { + if (pColSchema->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + return buildInvalidOperationMsg(&pBuf, "primary timestamp should not be NULL"); + } + + CHECK_CODE(MemRowAppend(&pBuf, NULL, 0, ¶m)); + } else { + int32_t colLen = pColSchema->bytes; + if (IS_VAR_DATA_TYPE(pColSchema->type)) { + colLen = bind->length[r]; + } + + CHECK_CODE(MemRowAppend(&pBuf, (char *)bind->buffer + bind->buffer_length * r, colLen, ¶m)); + } + + if (PRIMARYKEY_TIMESTAMP_COL_ID == pColSchema->colId) { + TSKEY tsKey = TD_ROW_KEY(row); + checkTimestamp(pDataBlock, (const char *)&tsKey); + } + + // set the null value for the columns that do not assign values + if (rowEnd && (spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { + for (int32_t i = 0; i < spd->numOfCols; ++i) { + if (spd->cols[i].valStat == VAL_STAT_NONE) { // the primary TS key is not VAL_STAT_NONE + tdAppendColValToTpRow(pBuilder, TD_VTYPE_NONE, getNullValue(pSchema[i].type), true, pSchema[i].type, i, + spd->cols[i].toffset); + } + } + } } - for (int32_t i = 0; i < boundInfo->numOfBound; ++i) { - SSchema* pTagSchema = &pSchema[boundInfo->boundColumns[i] - 1]; - strcpy((*fields)[i].name, pTagSchema->name); - (*fields)[i].type = pTagSchema->type; - (*fields)[i].bytes = pTagSchema->bytes; + if (rowEnd) { + pDataBlock->size += extendedRowSize * bind->num; + + SSubmitBlk *pBlocks = (SSubmitBlk *)(pDataBlock->pData); + if (TSDB_CODE_SUCCESS != setBlockInfo(pBlocks, pDataBlock, bind->num)) { + return buildInvalidOperationMsg(&pBuf, "too many rows in sql, total number of rows should be less than 32767"); + } + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t buildBoundFields(SParsedDataColInfo *boundInfo, SSchema *pSchema, int32_t *fieldNum, TAOS_FIELD** fields) { + if (fields) { + *fields = taosMemoryCalloc(boundInfo->numOfBound, sizeof(TAOS_FIELD)); + if (NULL == *fields) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < boundInfo->numOfBound; ++i) { + SSchema* pTagSchema = &pSchema[boundInfo->boundColumns[i] - 1]; + strcpy((*fields)[i].name, pTagSchema->name); + (*fields)[i].type = pTagSchema->type; + (*fields)[i].bytes = pTagSchema->bytes; + } } *fieldNum = boundInfo->numOfBound; @@ -1421,7 +1512,9 @@ int32_t qBuildStmtColFields(void *pBlock, int32_t *fieldNum, TAOS_FIELD** fields SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); if (pDataBlock->boundColumnInfo.numOfBound <= 0) { *fieldNum = 0; - *fields = NULL; + if (fields) { + *fields = NULL; + } return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 1674ef0ace..e2e2207c70 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -18,7 +18,11 @@ #include "parInt.h" #include "parToken.h" -static bool isInsertSql(const char* pStr, size_t length) { +bool isInsertSql(const char* pStr, size_t length) { + if (NULL == pStr) { + return false; + } + int32_t index = 0; do { diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 2bbb563d56..5a0610400c 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -34,15 +34,20 @@ typedef struct { int32_t gVarCharSize = 10; int32_t gVarCharLen = 5; +int32_t gExecLoopTimes = 1; // no change + + int insertMBSETest1(TAOS_STMT *stmt); int insertMBSETest2(TAOS_STMT *stmt); int insertMBMETest1(TAOS_STMT *stmt); int insertMBMETest2(TAOS_STMT *stmt); int insertMBMETest3(TAOS_STMT *stmt); int insertMBMETest4(TAOS_STMT *stmt); +int insertMPMETest1(TAOS_STMT *stmt); int32_t shortColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_INT}; int32_t longColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_BOOL, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_UTINYINT, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_USMALLINT, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_UINT, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_UBIGINT, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_BINARY, TSDB_DATA_TYPE_NCHAR}; +int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_FLOAT}; #define tListLen(x) (sizeof(x) / sizeof((x)[0])) @@ -50,7 +55,7 @@ int32_t longColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_BOOL, TSDB_DAT typedef struct { char caseDesc[128]; int32_t colNum; - int32_t *colList; + int32_t *colList; // full table column list bool autoCreate; bool fullCol; int32_t (*runFn)(TAOS_STMT*); @@ -66,7 +71,7 @@ CaseCfg gCase[] = { #if 0 {"insert:MBSE1-FULL", tListLen(shortColList), shortColList, false, true, insertMBSETest1, 1, 10, 10, 0, 0, 10}, {"insert:MBSE1-FULL", tListLen(shortColList), shortColList, false, true, insertMBSETest1, 10, 100, 10, 0, 0, 10}, - + {"insert:MBSE1-FULL", tListLen(longColList), longColList, false, true, insertMBSETest1, 10, 10, 2, 0, 0, 1}, {"insert:MBSE1-C012", tListLen(longColList), longColList, false, false, insertMBSETest1, 10, 10, 2, 12, 0, 1}, {"insert:MBSE1-C002", tListLen(longColList), longColList, false, false, insertMBSETest1, 10, 10, 2, 2, 0, 1}, @@ -86,17 +91,39 @@ CaseCfg gCase[] = { {"insert:MBME3-FULL", tListLen(longColList), longColList, false, true, insertMBMETest3, 10, 10, 2, 0, 0, 1}, {"insert:MBME3-C012", tListLen(longColList), longColList, false, false, insertMBMETest3, 10, 10, 2, 12, 0, 1}, {"insert:MBME3-C002", tListLen(longColList), longColList, false, false, insertMBMETest3, 10, 10, 2, 2, 0, 1}, + + {"insert:MBME4-FULL", tListLen(longColList), longColList, false, true, insertMBMETest4, 10, 10, 2, 0, 0, 1}, + {"insert:MBME4-C012", tListLen(longColList), longColList, false, false, insertMBMETest4, 10, 10, 2, 12, 0, 1}, + {"insert:MBME4-C002", tListLen(longColList), longColList, false, false, insertMBMETest4, 10, 10, 2, 2, 0, 1}, #endif -// {"insert:MBME4-FULL", tListLen(longColList), longColList, false, true, insertMBMETest4, 10, 10, 2, 0, 0, 1}, - {"insert:MBME4-C012", tListLen(longColList), longColList, false, false, insertMBMETest4, 10, 10, 2, 12, 0, 1}, -// {"insert:MBME4-C002", tListLen(longColList), longColList, false, false, insertMBMETest4, 10, 10, 2, 2, 0, 1}, + {"insert:MPME1-C012", tListLen(longColList), longColList, false, false, insertMPMETest1, 10, 10, 2, 12, 0, 1}, }; CaseCfg *gCurCase = NULL; -int32_t gTestNull = 1; -bool gTestAutoCreate = false; + +typedef struct { + int32_t bindNullNum; + bool autoCreate; + bool checkParamNum; + int32_t bindColNum; + int32_t bindRowNum; + int32_t bindColTypeNum; + int32_t* bindColTypeList; +} CaseCtrl; + +CaseCtrl gCaseCtrl = { + .bindNullNum = 0, + .autoCreate = false, + .bindColNum = 0, + .bindRowNum = 0, + .bindColTypeNum = 0, + .bindColTypeList = NULL, + .checkParamNum = false, +// .bindColTypeNum = tListLen(bindColTypeList), +// .bindColTypeList = bindColTypeList, +}; int32_t taosGetTimeOfDay(struct timeval *tv) { return gettimeofday(tv, NULL); @@ -217,6 +244,9 @@ void generateDataType(BindData *data, int32_t bindIdx, int32_t colIdx, int32_t * if (gCurCase->fullCol) { *dataType = gCurCase->colList[bindIdx]; return; + } else if (gCaseCtrl.bindColTypeNum) { + *dataType = gCaseCtrl.bindColTypeList[colIdx]; + return; } else if (0 == colIdx) { *dataType = TSDB_DATA_TYPE_TIMESTAMP; return; @@ -251,49 +281,49 @@ int32_t prepareColData(BindData *data, int32_t bindIdx, int32_t rowIdx, int32_t data->pBind[bindIdx].buffer_length = sizeof(bool); data->pBind[bindIdx].buffer = data->boolData + rowIdx; data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull + rowIdx; + data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_TINYINT: data->pBind[bindIdx].buffer_length = sizeof(int8_t); data->pBind[bindIdx].buffer = data->tinyData + rowIdx; data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull + rowIdx; + data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_SMALLINT: data->pBind[bindIdx].buffer_length = sizeof(int16_t); data->pBind[bindIdx].buffer = data->smallData + rowIdx; data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull + rowIdx; + data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_INT: data->pBind[bindIdx].buffer_length = sizeof(int32_t); data->pBind[bindIdx].buffer = data->intData + rowIdx; data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull + rowIdx; + data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_BIGINT: data->pBind[bindIdx].buffer_length = sizeof(int64_t); data->pBind[bindIdx].buffer = data->bigData + rowIdx; data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull + rowIdx; + data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_FLOAT: data->pBind[bindIdx].buffer_length = sizeof(float); data->pBind[bindIdx].buffer = data->floatData + rowIdx; data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull + rowIdx; + data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_DOUBLE: data->pBind[bindIdx].buffer_length = sizeof(double); data->pBind[bindIdx].buffer = data->doubleData + rowIdx; data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull + rowIdx; + data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_VARCHAR: data->pBind[bindIdx].buffer_length = gVarCharSize; data->pBind[bindIdx].buffer = data->binaryData + rowIdx * gVarCharSize; data->pBind[bindIdx].length = data->binaryLen; - data->pBind[bindIdx].is_null = data->isNull + rowIdx; + data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_TIMESTAMP: data->pBind[bindIdx].buffer_length = sizeof(int64_t); @@ -305,31 +335,31 @@ int32_t prepareColData(BindData *data, int32_t bindIdx, int32_t rowIdx, int32_t data->pBind[bindIdx].buffer_length = gVarCharSize; data->pBind[bindIdx].buffer = data->binaryData + rowIdx * gVarCharSize; data->pBind[bindIdx].length = data->binaryLen; - data->pBind[bindIdx].is_null = data->isNull + rowIdx; + data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_UTINYINT: data->pBind[bindIdx].buffer_length = sizeof(uint8_t); data->pBind[bindIdx].buffer = data->utinyData + rowIdx; data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull + rowIdx; + data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_USMALLINT: data->pBind[bindIdx].buffer_length = sizeof(uint16_t); data->pBind[bindIdx].buffer = data->usmallData + rowIdx; data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull + rowIdx; + data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_UINT: data->pBind[bindIdx].buffer_length = sizeof(uint32_t); data->pBind[bindIdx].buffer = data->uintData + rowIdx; data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull + rowIdx; + data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_UBIGINT: data->pBind[bindIdx].buffer_length = sizeof(uint64_t); data->pBind[bindIdx].buffer = data->ubigData + rowIdx; data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull + rowIdx; + data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; default: printf("invalid col type:%d", dataType); @@ -417,6 +447,69 @@ void destroyData(BindData *data) { taosMemoryFree(data->binaryLen); taosMemoryFree(data->isNull); taosMemoryFree(data->pBind); + taosMemoryFree(data->colTypes); +} + +int32_t bpBindParam(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) { + static int32_t n = 0; + + if (gCurCase->bindRowNum > 1) { + if (0 == (n++%2)) { + if (taos_stmt_bind_param_batch(stmt, bind)) { + printf("taos_stmt_bind_param_batch error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } else { + for (int32_t i = 0; i < gCurCase->bindColNum; ++i) { + if (taos_stmt_bind_single_param_batch(stmt, bind++, i)) { + printf("taos_stmt_bind_single_param_batch error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } + } + } else { + if (taos_stmt_bind_param(stmt, bind)) { + printf("taos_stmt_bind_param error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } + + return 0; +} + +void bpCheckIsInsert(TAOS_STMT *stmt) { + int32_t isInsert = 0; + if (taos_stmt_is_insert(stmt, &isInsert)) { + printf("taos_stmt_is_insert error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + if (0 == isInsert) { + printf("is insert failed\n"); + exit(1); + } +} + +void bpCheckParamNum(TAOS_STMT *stmt) { + int32_t num = 0; + if (taos_stmt_num_params(stmt, &num)) { + printf("taos_stmt_num_params error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + if (gCurCase->bindColNum != num) { + printf("is insert failed\n"); + exit(1); + } +} + +void bpCheckAffectedRows(TAOS_STMT *stmt, int32_t times) { + int32_t rows = taos_stmt_affected_rows(stmt); + int32_t insertNum = gCurCase->rowNum * gCurCase->tblNum * times; + if (insertNum != rows) { + printf("affected rows %d mis-match with insert num %d\n", rows, insertNum); + exit(1); + } } @@ -433,6 +526,8 @@ int insertMBSETest1(TAOS_STMT *stmt) { exit(1); } + bpCheckIsInsert(stmt); + int32_t bindTimes = gCurCase->rowNum/gCurCase->bindRowNum; for (int32_t t = 0; t< gCurCase->tblNum; ++t) { if (gCurCase->tblNum > 1) { @@ -444,10 +539,13 @@ int insertMBSETest1(TAOS_STMT *stmt) { exit(1); } } + + if (gCaseCtrl.checkParamNum) { + bpCheckParamNum(stmt); + } for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { - printf("taos_stmt_bind_param error:%s\n", taos_stmt_errstr(stmt)); + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { exit(1); } @@ -463,6 +561,11 @@ int insertMBSETest1(TAOS_STMT *stmt) { exit(1); } + bpCheckIsInsert(stmt); + bpCheckAffectedRows(stmt, 1); + + destroyData(&data); + return 0; } @@ -480,6 +583,8 @@ int insertMBSETest2(TAOS_STMT *stmt) { exit(1); } + bpCheckIsInsert(stmt); + int32_t bindTimes = gCurCase->rowNum/gCurCase->bindRowNum; for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { - printf("taos_stmt_bind_param error:%s\n", taos_stmt_errstr(stmt)); + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { exit(1); } - + + if (gCaseCtrl.checkParamNum) { + bpCheckParamNum(stmt); + } + if (taos_stmt_add_batch(stmt)) { printf("taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); exit(1); @@ -511,6 +619,11 @@ int insertMBSETest2(TAOS_STMT *stmt) { exit(1); } + bpCheckIsInsert(stmt); + bpCheckAffectedRows(stmt, 1); + + destroyData(&data); + return 0; } @@ -527,6 +640,8 @@ int insertMBMETest1(TAOS_STMT *stmt) { exit(1); } + bpCheckIsInsert(stmt); + int32_t bindTimes = gCurCase->rowNum/gCurCase->bindRowNum; for (int32_t t = 0; t< gCurCase->tblNum; ++t) { if (gCurCase->tblNum > 1) { @@ -538,10 +653,13 @@ int insertMBMETest1(TAOS_STMT *stmt) { exit(1); } } + + if (gCaseCtrl.checkParamNum) { + bpCheckParamNum(stmt); + } for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { - printf("taos_stmt_bind_param error:%s\n", taos_stmt_errstr(stmt)); + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { exit(1); } @@ -557,6 +675,10 @@ int insertMBMETest1(TAOS_STMT *stmt) { } } + bpCheckIsInsert(stmt); + bpCheckAffectedRows(stmt, 1); + + destroyData(&data); return 0; } @@ -574,6 +696,8 @@ int insertMBMETest2(TAOS_STMT *stmt) { exit(1); } + bpCheckIsInsert(stmt); + int32_t bindTimes = gCurCase->rowNum/gCurCase->bindRowNum; for (int32_t t = 0; t< gCurCase->tblNum; ++t) { if (gCurCase->tblNum > 1) { @@ -587,11 +711,14 @@ int insertMBMETest2(TAOS_STMT *stmt) { } for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { - printf("taos_stmt_bind_param error:%s\n", taos_stmt_errstr(stmt)); + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { exit(1); } - + + if (gCaseCtrl.checkParamNum) { + bpCheckParamNum(stmt); + } + if (taos_stmt_add_batch(stmt)) { printf("taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); exit(1); @@ -604,6 +731,10 @@ int insertMBMETest2(TAOS_STMT *stmt) { } } + bpCheckIsInsert(stmt); + bpCheckAffectedRows(stmt, 1); + + destroyData(&data); return 0; } @@ -621,6 +752,8 @@ int insertMBMETest3(TAOS_STMT *stmt) { exit(1); } + bpCheckIsInsert(stmt); + int32_t bindTimes = gCurCase->rowNum/gCurCase->bindRowNum; for (int32_t t = 0; t< gCurCase->tblNum; ++t) { if (gCurCase->tblNum > 1) { @@ -632,6 +765,10 @@ int insertMBMETest3(TAOS_STMT *stmt) { exit(1); } } + + if (gCaseCtrl.checkParamNum) { + bpCheckParamNum(stmt); + } for (int32_t b = 0; b tblNum > 1) { @@ -644,8 +781,7 @@ int insertMBMETest3(TAOS_STMT *stmt) { } } - if (taos_stmt_bind_param_batch(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { - printf("taos_stmt_bind_param error:%s\n", taos_stmt_errstr(stmt)); + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { exit(1); } @@ -661,6 +797,10 @@ int insertMBMETest3(TAOS_STMT *stmt) { } } + bpCheckIsInsert(stmt); + bpCheckAffectedRows(stmt, 1); + + destroyData(&data); return 0; } @@ -679,6 +819,8 @@ int insertMBMETest4(TAOS_STMT *stmt) { exit(1); } + bpCheckIsInsert(stmt); + int32_t bindTimes = gCurCase->rowNum/gCurCase->bindRowNum; for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { - printf("taos_stmt_bind_param error:%s\n", taos_stmt_errstr(stmt)); + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { exit(1); } - + + if (gCaseCtrl.checkParamNum) { + bpCheckParamNum(stmt); + } + if (taos_stmt_add_batch(stmt)) { printf("taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); exit(1); @@ -710,9 +855,81 @@ int insertMBMETest4(TAOS_STMT *stmt) { } } + bpCheckIsInsert(stmt); + bpCheckAffectedRows(stmt, 1); + + destroyData(&data); + return 0; } +/* [prepare [settbname [bind add] exec]] */ +int insertMPMETest1(TAOS_STMT *stmt) { + int32_t loop = 0; + + while (gCurCase->bindColNum >= 2) { + BindData data = {0}; + prepareData(&data); + + printf("SQL: %s\n", data.sql); + + int code = taos_stmt_prepare(stmt, data.sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + bpCheckIsInsert(stmt); + + int32_t bindTimes = gCurCase->rowNum/gCurCase->bindRowNum; + for (int32_t t = 0; t< gCurCase->tblNum; ++t) { + if (gCurCase->tblNum > 1) { + char buf[32]; + sprintf(buf, "t%d", t); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } + + if (gCaseCtrl.checkParamNum) { + bpCheckParamNum(stmt); + } + + for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { + exit(1); + } + + if (taos_stmt_add_batch(stmt)) { + printf("taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } + + if (taos_stmt_execute(stmt) != 0) { + printf("taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } + + bpCheckIsInsert(stmt); + + destroyData(&data); + + gCurCase->bindColNum -= 2; + loop++; + } + + bpCheckAffectedRows(stmt, loop); + + gExecLoopTimes = loop; + + return 0; +} + + #if 0 int stmt_func1(TAOS_STMT *stmt) { struct { @@ -4015,7 +4232,7 @@ void prepareCheckResult(TAOS *taos) { sprintf(buf, "t%d", 0); } - prepareCheckResultImpl(taos, buf, 1, gCurCase->rowNum); + prepareCheckResultImpl(taos, buf, 1, gCurCase->rowNum * gExecLoopTimes); } } @@ -4347,8 +4564,17 @@ void* runcase(TAOS *taos) { gCurCase->bindColNum = gCurCase->colNum; } - gCurCase->bindNullNum = gTestNull; - gCurCase->autoCreate = gTestAutoCreate; + gCurCase->bindNullNum = gCaseCtrl.bindNullNum; + gCurCase->autoCreate = gCaseCtrl.autoCreate; + if (gCaseCtrl.bindColNum) { + gCurCase->bindColNum = gCaseCtrl.bindColNum; + } + if (gCaseCtrl.bindRowNum) { + gCurCase->bindRowNum = gCaseCtrl.bindRowNum; + } + if (gCaseCtrl.bindColTypeNum) { + gCurCase->bindRowNum = gCaseCtrl.bindColTypeNum; + } for (int32_t n = 0; n < gCurCase->runTimes; ++n) { prepare(taos, gCurCase->colNum, gCurCase->colList, gCurCase->autoCreate); @@ -4362,6 +4588,8 @@ void* runcase(TAOS *taos) { (*gCurCase->runFn)(stmt); prepareCheckResult(taos); + + taos_stmt_close(stmt); } printf("* Case %d - %s End *\n", i, gCurCase->caseDesc); From 1ab6342018927dfcbce4c7f30f7f3ec0c08a44ca Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 20 Apr 2022 18:54:15 +0800 Subject: [PATCH 35/47] refactor: fix error in merge --- source/libs/parser/src/parUtil.c | 1 - source/libs/scalar/src/sclvector.c | 1 - tools/shell/src/shellEngine.c | 3 ++- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 0ff5843736..7f945eb70c 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -262,7 +262,6 @@ int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen) { j++; } dst[j] = '\0'; - strtrim(dst); return j; } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index ae8c395997..d484738328 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -287,7 +287,6 @@ static FORCE_INLINE void varToNchar(char* buf, SScalarParam* pOut, int32_t rowIn //TODO opt performance, tmp is not needed. int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, int32_t inType, int32_t outType) { - ASSERT(!IS_VAR_DATA_TYPE(outType)); int32_t bufSize = pIn->columnData->info.bytes; char *tmp = taosMemoryMalloc(bufSize + VARSTR_HEADER_SIZE); diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index e8f53ef981..ca5bf69795 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -761,7 +761,8 @@ static int calcColWidth(TAOS_FIELD *field, int precision) { return TMAX(field->bytes, width); } - case TSDB_DATA_TYPE_NCHAR: { + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_JSON:{ int16_t bytes = field->bytes * TSDB_NCHAR_SIZE; if (bytes > tsMaxBinaryDisplayWidth) { return TMAX(tsMaxBinaryDisplayWidth, width); From 32c24eedb87a247478327ea88170512e23c68356 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 20 Apr 2022 19:05:00 +0800 Subject: [PATCH 36/47] fix: invalid comment len --- source/dnode/mnode/impl/src/mndStb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index d8b94f5ffe..6bdf4575a8 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -348,7 +348,7 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) { memcpy(pOld->pColumns, pNew->pColumns, pOld->numOfColumns * sizeof(SSchema)); memcpy(pOld->pTags, pNew->pTags, pOld->numOfTags * sizeof(SSchema)); if (pNew->commentLen != 0) { - memcpy(pOld->comment, pNew->comment, TSDB_STB_COMMENT_LEN); + memcpy(pOld->comment, pNew->comment, pNew->commentLen); } if (pNew->ast1Len != 0) { memcpy(pOld->pAst1, pNew->pAst1, pNew->ast1Len); From ca34398c436fb4323750de6a0fb5229ac1526526 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 20 Apr 2022 19:27:39 +0800 Subject: [PATCH 37/47] prepare for dnode start/stop/restart udfd and vnode/snode/qnode split --- source/libs/function/inc/tudf.h | 22 ++++++++++++++++------ source/libs/function/inc/udfc.h | 4 ++-- source/libs/function/src/tudf.c | 25 +++++++++++-------------- source/libs/function/src/udfd.c | 10 +++------- source/libs/function/test/runUdf.c | 4 ++-- 5 files changed, 34 insertions(+), 31 deletions(-) diff --git a/source/libs/function/inc/tudf.h b/source/libs/function/inc/tudf.h index d7bb6dfb46..c51b6e1264 100644 --- a/source/libs/function/inc/tudf.h +++ b/source/libs/function/inc/tudf.h @@ -38,17 +38,28 @@ enum { UDFC_CODE_PIPE_READ_ERR = -3, }; +/*TODO: no api for dnode startudfd/stopudfd*/ /** - * start udf dameon service - * @return error code + * start udfd dameon service */ -int32_t startUdfService(); +int32_t startUdfd(int32_t dnodeId); /** - * stop udf dameon service + * stop udfd dameon service + */ +int32_t stopUdfd(int32_t dnodeId); + +/** + * create udfd proxy, called once in process that call setupUdf/callUdfxxx/teardownUdf * @return error code */ -int32_t stopUdfService(); +int32_t createUdfdProxy(int32_t dnodeId); + +/** + * destroy udfd proxy + * @return error code + */ +int32_t destroyUdfdProxy(int32_t dnodeId); typedef void *UdfHandle; @@ -104,7 +115,6 @@ typedef struct SUdfInterBuf { char* buf; } SUdfInterBuf; -//TODO: translate these calls to callUdf // output: interBuf int32_t callUdfAggInit(UdfHandle handle, SUdfInterBuf *interBuf); // input: block, state diff --git a/source/libs/function/inc/udfc.h b/source/libs/function/inc/udfc.h index 4d2aeb7049..fed2818ced 100644 --- a/source/libs/function/inc/udfc.h +++ b/source/libs/function/inc/udfc.h @@ -32,9 +32,9 @@ typedef struct SUdfInfo { typedef void *UdfHandle; -int32_t startUdfService(); +int32_t createUdfdProxy(); -int32_t stopUdfService(); +int32_t destroyUdfdProxy(); //int32_t setupUdf(SUdfInfo *udf, int32_t numOfUdfs, UdfHandle *handles); diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index c41447b584..ad74daddc6 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -200,10 +200,10 @@ int64_t gUdfTaskSeqNum = 0; enum { UDFC_STATE_INITAL = 0, // initial state - UDFC_STATE_STARTNG, // starting after startUdfService + UDFC_STATE_STARTNG, // starting after createUdfdProxy UDFC_STATE_READY, // started and begin to receive quests UDFC_STATE_RESTARTING, // udfd abnormal exit. cleaning up and restart. - UDFC_STATE_STOPPING, // stopping after stopUdfService + UDFC_STATE_STOPPING, // stopping after destroyUdfdProxy UDFC_STATUS_FINAL, // stopped }; int8_t gUdfcState = UDFC_STATE_INITAL; @@ -929,7 +929,7 @@ void udfStopAsyncCb(uv_async_t *async) { } } -int32_t startUdfd(); +int32_t udfcSpawnUdfd(); void onUdfdExit(uv_process_t *req, int64_t exit_status, int term_signal) { //TODO: pipe close will be first received @@ -944,12 +944,12 @@ void onUdfdExit(uv_process_t *req, int64_t exit_status, int term_signal) { if (gUdfcState == UDFC_STATE_READY) { gUdfcState = UDFC_STATE_RESTARTING; //TODO: asynchronous without blocking. how to do it - cleanUpUvTasks(); - startUdfd(); + //cleanUpUvTasks(); + udfcSpawnUdfd(); } } -int32_t startUdfd() { +int32_t udfcSpawnUdfd() { //TODO: path uv_process_options_t options = {0}; static char path[256] = {0}; @@ -979,9 +979,6 @@ int32_t startUdfd() { void constructUdfService(void *argsThread) { uv_loop_init(&gUdfdLoop); - //TODO spawn error - startUdfd(); - uv_async_init(&gUdfdLoop, &gUdfLoopTaskAync, udfClientAsyncCb); uv_async_init(&gUdfdLoop, &gUdfLoopStopAsync, udfStopAsyncCb); uv_mutex_init(&gUdfTaskQueueMutex); @@ -994,7 +991,7 @@ void constructUdfService(void *argsThread) { } -int32_t startUdfService() { +int32_t createUdfdProxy(int32_t dnodeId) { gUdfcState = UDFC_STATE_STARTNG; uv_barrier_init(&gUdfInitBarrier, 2); uv_thread_create(&gUdfLoopThread, constructUdfService, 0); @@ -1002,12 +999,12 @@ int32_t startUdfService() { return 0; } -int32_t stopUdfService() { +int32_t destroyUdfdProxy(int32_t dnodeId) { gUdfcState = UDFC_STATE_STOPPING; uv_barrier_destroy(&gUdfInitBarrier); - if (gUdfcState == UDFC_STATE_STOPPING) { - uv_process_kill(&gUdfdProcess, SIGINT); - } +// if (gUdfcState == UDFC_STATE_STOPPING) { +// uv_process_kill(&gUdfdProcess, SIGINT); +// } uv_async_send(&gUdfLoopStopAsync); uv_thread_join(&gUdfLoopThread); uv_mutex_destroy(&gUdfTaskQueueMutex); diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 641b8535c3..71434c695f 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -83,18 +83,14 @@ typedef struct SUdfHandle { int32_t udfdLoadUdf(char* udfName, SUdf* udf) { strcpy(udf->name, udfName); - // TODO: retrive udf info from mnode - char *path = "libudf1.so"; - int err = uv_dlopen(path, &udf->lib); + int err = uv_dlopen(udf->path, &udf->lib); if (err != 0) { - fnError("can not load library %s. error: %s", path, uv_strerror(err)); + fnError("can not load library %s. error: %s", udf->path, uv_strerror(err)); // TODO set error } - + //TODO: find all the functions char normalFuncName[TSDB_FUNC_NAME_LEN] = {0}; strcpy(normalFuncName, udfName); - // TODO error, multi-thread, same udf, lock it - // TODO find all functions normal, init, destroy, normal, merge, finalize uv_dlsym(&udf->lib, normalFuncName, (void **)(&udf->scalarProcFunc)); char freeFuncName[TSDB_FUNC_NAME_LEN + 6] = {0}; char *freeSuffix = "_free"; diff --git a/source/libs/function/test/runUdf.c b/source/libs/function/test/runUdf.c index 28dc6bb99a..41c7f65e0b 100644 --- a/source/libs/function/test/runUdf.c +++ b/source/libs/function/test/runUdf.c @@ -8,7 +8,7 @@ #include "tdatablock.h" int main(int argc, char *argv[]) { - startUdfService(); + createUdfdProxy(1); uv_sleep(1000); char path[256] = {0}; size_t cwdSize = 256; @@ -53,5 +53,5 @@ int main(int argc, char *argv[]) { } teardownUdf(handle); - stopUdfService(); + destroyUdfdProxy(1); } From da232cbeaf2a906fe9c8cf74537cb869b6543baa Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Wed, 20 Apr 2022 20:02:10 +0800 Subject: [PATCH 38/47] [test: check error from valgrind log file] --- tests/script/sh/checkValgrind.sh | 38 ++++++++++++ tests/script/tsim/valgrind/checkError.sim | 76 +++++++++++++++++++++++ tests/tsim/src/simExe.c | 6 ++ 3 files changed, 120 insertions(+) create mode 100755 tests/script/sh/checkValgrind.sh create mode 100644 tests/script/tsim/valgrind/checkError.sim diff --git a/tests/script/sh/checkValgrind.sh b/tests/script/sh/checkValgrind.sh new file mode 100755 index 0000000000..17d7ce1aff --- /dev/null +++ b/tests/script/sh/checkValgrind.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +set +e +#set -x + +NODE_NAME= + +while getopts "n:" arg +do + case $arg in + n) + NODE_NAME=$OPTARG + ;; + ?) + echo "unkown argument" + ;; + esac +done + +SCRIPT_DIR=`dirname $0` +cd $SCRIPT_DIR/../ +SCRIPT_DIR=`pwd` + +IN_TDINTERNAL="community" +if [[ "$SCRIPT_DIR" == *"$IN_TDINTERNAL"* ]]; then + cd ../../.. +else + cd ../../ +fi + +TAOS_DIR=`pwd` +LOG_DIR=$TAOS_DIR/sim/$NODE_NAME/log +#CFG_DIR=$TAOS_DIR/sim/$NODE_NAME/cfg + +#echo ---- $LOG_DIR + +errors=`grep "ERROR SUMMARY:" ${LOG_DIR}/valgrind-taosd-*.log | cut -d ' ' -f 2,3,4,5 | tr -d "\n"` +echo $errors diff --git a/tests/script/tsim/valgrind/checkError.sim b/tests/script/tsim/valgrind/checkError.sim new file mode 100644 index 0000000000..3b66cb4a8c --- /dev/null +++ b/tests/script/tsim/valgrind/checkError.sim @@ -0,0 +1,76 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +#system sh/deploy.sh -n dnode2 -i 2 +#system sh/deploy.sh -n dnode3 -i 3 +#system sh/deploy.sh -n dnode4 -i 4 +#system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 +system sh/exec.sh -n dnode1 -s start +#system sh/exec.sh -n dnode2 -s start +#system sh/exec.sh -n dnode3 -s start +#system sh/exec.sh -n dnode4 -s start + +sleep 2000 + +#$loop_cnt = 0 +#check_dnode_ready: +# $loop_cnt = $loop_cnt + 1 +# sleep 200 +# if $loop_cnt == 10 then +# print ====> dnode not ready! +# return -1 +# endi +#sql show dnodes +#print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +#print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +#print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] +#print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] +#if $data[0][0] != 1 then +# return -1 +#endi +#if $data[0][4] != ready then +# goto check_dnode_ready +#endi +# +##sql connect +#sql create dnode $hostname port 7200 +#sql create dnode $hostname port 7300 +#sql create dnode $hostname port 7400 +# +#$loop_cnt = 0 +#check_dnode_ready_1: +#$loop_cnt = $loop_cnt + 1 +#sleep 200 +#if $loop_cnt == 10 then +# print ====> dnodes not ready! +# return -1 +#endi +#sql show dnodes +#print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +#print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +#print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] +#print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] +#if $data[0][4] != ready then +# goto check_dnode_ready_1 +#endi +#if $data[1][4] != ready then +# goto check_dnode_ready_1 +#endi +#if $data[2][4] != ready then +# goto check_dnode_ready_1 +#endi +#if $data[3][4] != ready then +# goto check_dnode_ready_1 +#endi + +#=========== please add any actions above ================= + +print ====> stop all dondes to output valgrind log file +system sh/exec.sh -n dnode1 -s stop -x SIGINT + +print ====> start to check if there are ERRORS in vagrind log file for each dnode +# -n : dnode[x] be check +system_content sh/checkValgrind.sh -n dnode1 +print cmd return result----> [ $system_content ] +if $system_content != @ERROR SUMMARY: 0 errors@ then + return -1 +endi diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index 8b3edb6035..dbc3c2c460 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -441,6 +441,12 @@ void simStoreSystemContentResult(SScript *script, char *filename) { // if ((fd = fopen(filename, "r")) != NULL) { if ((pFile = taosOpenFile(filename, TD_FILE_READ)) != NULL) { taosReadFile(pFile, script->system_ret_content, MAX_SYSTEM_RESULT_LEN - 1); + int32_t len = strlen(script->system_ret_content); + for (int32_t i = 0; i < len; ++i) { + if (script->system_ret_content[i] == '\n' || script->system_ret_content[i] == '\r') { + script->system_ret_content[i] = 0; + } + } taosCloseFile(&pFile); char rmCmd[MAX_FILE_NAME_LEN] = {0}; sprintf(rmCmd, "rm -f %s", filename); From dff638c3770450932c96ae45ccff850c5d5aa4ee Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 20 Apr 2022 20:07:49 +0800 Subject: [PATCH 39/47] stmt --- tests/script/api/batchprepare.c | 1088 +++---------------------------- 1 file changed, 104 insertions(+), 984 deletions(-) diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 5a0610400c..7fdcfcd167 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -9,6 +9,12 @@ #include #include "../../../include/client/taos.h" +int32_t shortColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_INT}; +int32_t fullColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_BOOL, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_UTINYINT, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_USMALLINT, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_UINT, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_UBIGINT, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_BINARY, TSDB_DATA_TYPE_NCHAR}; +int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_FLOAT}; + +#define tListLen(x) (sizeof(x) / sizeof((x)[0])) + typedef struct { int64_t* tsData; bool* boolData; @@ -35,7 +41,7 @@ int32_t gVarCharSize = 10; int32_t gVarCharLen = 5; int32_t gExecLoopTimes = 1; // no change - +int32_t gFullColNum = tListLen(fullColList); int insertMBSETest1(TAOS_STMT *stmt); int insertMBSETest2(TAOS_STMT *stmt); @@ -45,11 +51,7 @@ int insertMBMETest3(TAOS_STMT *stmt); int insertMBMETest4(TAOS_STMT *stmt); int insertMPMETest1(TAOS_STMT *stmt); -int32_t shortColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_INT}; -int32_t longColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_BOOL, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_UTINYINT, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_USMALLINT, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_UINT, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_UBIGINT, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_BINARY, TSDB_DATA_TYPE_NCHAR}; -int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_FLOAT}; -#define tListLen(x) (sizeof(x) / sizeof((x)[0])) typedef struct { @@ -69,35 +71,36 @@ typedef struct { CaseCfg gCase[] = { #if 0 - {"insert:MBSE1-FULL", tListLen(shortColList), shortColList, false, true, insertMBSETest1, 1, 10, 10, 0, 0, 10}, - {"insert:MBSE1-FULL", tListLen(shortColList), shortColList, false, true, insertMBSETest1, 10, 100, 10, 0, 0, 10}, + {"insert:MBSE1-FULL", tListLen(shortColList), shortColList, false, true, insertMBSETest1, 1, 10, 10, 0, 0, 1}, + {"insert:MBSE1-FULL", tListLen(shortColList), shortColList, false, true, insertMBSETest1, 10, 100, 10, 0, 0, 1}, - {"insert:MBSE1-FULL", tListLen(longColList), longColList, false, true, insertMBSETest1, 10, 10, 2, 0, 0, 1}, - {"insert:MBSE1-C012", tListLen(longColList), longColList, false, false, insertMBSETest1, 10, 10, 2, 12, 0, 1}, - {"insert:MBSE1-C002", tListLen(longColList), longColList, false, false, insertMBSETest1, 10, 10, 2, 2, 0, 1}, + {"insert:MBSE1-FULL", tListLen(fullColList), fullColList, false, true, insertMBSETest1, 10, 10, 2, 0, 0, 1}, + {"insert:MBSE1-C012", tListLen(fullColList), fullColList, false, false, insertMBSETest1, 10, 10, 2, 12, 0, 1}, + {"insert:MBSE1-C002", tListLen(fullColList), fullColList, false, false, insertMBSETest1, 10, 10, 2, 2, 0, 1}, - {"insert:MBSE2-FULL", tListLen(longColList), longColList, false, true, insertMBSETest2, 10, 10, 2, 0, 0, 1}, - {"insert:MBSE2-C012", tListLen(longColList), longColList, false, false, insertMBSETest2, 10, 10, 2, 12, 0, 1}, - {"insert:MBSE2-C002", tListLen(longColList), longColList, false, false, insertMBSETest2, 10, 10, 2, 2, 0, 1}, + {"insert:MBSE2-FULL", tListLen(fullColList), fullColList, false, true, insertMBSETest2, 10, 10, 2, 0, 0, 1}, + {"insert:MBSE2-C012", tListLen(fullColList), fullColList, false, false, insertMBSETest2, 10, 10, 2, 12, 0, 1}, + {"insert:MBSE2-C002", tListLen(fullColList), fullColList, false, false, insertMBSETest2, 10, 10, 2, 2, 0, 1}, - {"insert:MBME1-FULL", tListLen(longColList), longColList, false, true, insertMBMETest1, 10, 10, 2, 0, 0, 1}, - {"insert:MBME1-C012", tListLen(longColList), longColList, false, false, insertMBMETest1, 10, 10, 2, 12, 0, 1}, - {"insert:MBME1-C002", tListLen(longColList), longColList, false, false, insertMBMETest1, 10, 10, 2, 2, 0, 1}, + {"insert:MBME1-FULL", tListLen(fullColList), fullColList, false, true, insertMBMETest1, 10, 10, 2, 0, 0, 1}, + {"insert:MBME1-C012", tListLen(fullColList), fullColList, false, false, insertMBMETest1, 10, 10, 2, 12, 0, 1}, + {"insert:MBME1-C002", tListLen(fullColList), fullColList, false, false, insertMBMETest1, 10, 10, 2, 2, 0, 1}, - {"insert:MBME2-FULL", tListLen(longColList), longColList, false, true, insertMBMETest2, 10, 10, 2, 0, 0, 1}, - {"insert:MBME2-C012", tListLen(longColList), longColList, false, false, insertMBMETest2, 10, 10, 2, 12, 0, 1}, - {"insert:MBME2-C002", tListLen(longColList), longColList, false, false, insertMBMETest2, 10, 10, 2, 2, 0, 1}, + {"insert:MBME2-FULL", tListLen(fullColList), fullColList, false, true, insertMBMETest2, 10, 10, 2, 0, 0, 1}, + {"insert:MBME2-C012", tListLen(fullColList), fullColList, false, false, insertMBMETest2, 10, 10, 2, 12, 0, 1}, + {"insert:MBME2-C002", tListLen(fullColList), fullColList, false, false, insertMBMETest2, 10, 10, 2, 2, 0, 1}, - {"insert:MBME3-FULL", tListLen(longColList), longColList, false, true, insertMBMETest3, 10, 10, 2, 0, 0, 1}, - {"insert:MBME3-C012", tListLen(longColList), longColList, false, false, insertMBMETest3, 10, 10, 2, 12, 0, 1}, - {"insert:MBME3-C002", tListLen(longColList), longColList, false, false, insertMBMETest3, 10, 10, 2, 2, 0, 1}, + {"insert:MBME3-FULL", tListLen(fullColList), fullColList, false, true, insertMBMETest3, 10, 10, 2, 0, 0, 1}, + {"insert:MBME3-C012", tListLen(fullColList), fullColList, false, false, insertMBMETest3, 10, 10, 2, 12, 0, 1}, + {"insert:MBME3-C002", tListLen(fullColList), fullColList, false, false, insertMBMETest3, 10, 10, 2, 2, 0, 1}, - {"insert:MBME4-FULL", tListLen(longColList), longColList, false, true, insertMBMETest4, 10, 10, 2, 0, 0, 1}, - {"insert:MBME4-C012", tListLen(longColList), longColList, false, false, insertMBMETest4, 10, 10, 2, 12, 0, 1}, - {"insert:MBME4-C002", tListLen(longColList), longColList, false, false, insertMBMETest4, 10, 10, 2, 2, 0, 1}, + {"insert:MBME4-FULL", tListLen(fullColList), fullColList, false, true, insertMBMETest4, 10, 10, 2, 0, 0, 1}, + {"insert:MBME4-C012", tListLen(fullColList), fullColList, false, false, insertMBMETest4, 10, 10, 2, 12, 0, 1}, + {"insert:MBME4-C002", tListLen(fullColList), fullColList, false, false, insertMBMETest4, 10, 10, 2, 2, 0, 1}, #endif - {"insert:MPME1-C012", tListLen(longColList), longColList, false, false, insertMPMETest1, 10, 10, 2, 12, 0, 1}, + {"insert:MPME1-FULL", tListLen(fullColList), fullColList, false, true, insertMPMETest1, 10, 10, 2, 0, 0, 1}, +// {"insert:MPME1-C012", tListLen(fullColList), fullColList, false, false, insertMPMETest1, 10, 10, 2, 12, 0, 1}, }; @@ -107,22 +110,26 @@ typedef struct { int32_t bindNullNum; bool autoCreate; bool checkParamNum; + bool printRes; + int32_t rowNum; //row num for one table int32_t bindColNum; - int32_t bindRowNum; + int32_t bindRowNum; //row num for once bind int32_t bindColTypeNum; int32_t* bindColTypeList; + int32_t runTimes; } CaseCtrl; CaseCtrl gCaseCtrl = { .bindNullNum = 0, .autoCreate = false, + .rowNum = 0, .bindColNum = 0, .bindRowNum = 0, .bindColTypeNum = 0, .bindColTypeList = NULL, .checkParamNum = false, -// .bindColTypeNum = tListLen(bindColTypeList), -// .bindColTypeList = bindColTypeList, + .printRes = true, + .runTimes = 0, }; int32_t taosGetTimeOfDay(struct timeval *tv) { @@ -241,12 +248,12 @@ void generateInsertSQL(BindData *data) { void generateDataType(BindData *data, int32_t bindIdx, int32_t colIdx, int32_t *dataType) { if (bindIdx < gCurCase->bindColNum) { - if (gCurCase->fullCol) { - *dataType = gCurCase->colList[bindIdx]; - return; - } else if (gCaseCtrl.bindColTypeNum) { + if (gCaseCtrl.bindColTypeNum) { *dataType = gCaseCtrl.bindColTypeList[colIdx]; return; + } else if (gCurCase->fullCol) { + *dataType = gCurCase->colList[bindIdx]; + return; } else if (0 == colIdx) { *dataType = TSDB_DATA_TYPE_TIMESTAMP; return; @@ -919,6 +926,7 @@ int insertMPMETest1(TAOS_STMT *stmt) { destroyData(&data); gCurCase->bindColNum -= 2; + gCurCase->fullCol = false; loop++; } @@ -931,379 +939,6 @@ int insertMPMETest1(TAOS_STMT *stmt) { #if 0 -int stmt_func1(TAOS_STMT *stmt) { - struct { - int64_t ts; - int8_t b; - int8_t v1; - int16_t v2; - int32_t v4; - int64_t v8; - float f4; - double f8; - char bin[40]; - char blob[80]; - } v = {0}; - - TAOS_BIND params[10]; - params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[0].buffer_length = sizeof(v.ts); - params[0].buffer = &v.ts; - params[0].length = ¶ms[0].buffer_length; - params[0].is_null = NULL; - - params[1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[1].buffer_length = sizeof(v.b); - params[1].buffer = &v.b; - params[1].length = ¶ms[1].buffer_length; - params[1].is_null = NULL; - - params[2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[2].buffer_length = sizeof(v.v1); - params[2].buffer = &v.v1; - params[2].length = ¶ms[2].buffer_length; - params[2].is_null = NULL; - - params[3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[3].buffer_length = sizeof(v.v2); - params[3].buffer = &v.v2; - params[3].length = ¶ms[3].buffer_length; - params[3].is_null = NULL; - - params[4].buffer_type = TSDB_DATA_TYPE_INT; - params[4].buffer_length = sizeof(v.v4); - params[4].buffer = &v.v4; - params[4].length = ¶ms[4].buffer_length; - params[4].is_null = NULL; - - params[5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[5].buffer_length = sizeof(v.v8); - params[5].buffer = &v.v8; - params[5].length = ¶ms[5].buffer_length; - params[5].is_null = NULL; - - params[6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[6].buffer_length = sizeof(v.f4); - params[6].buffer = &v.f4; - params[6].length = ¶ms[6].buffer_length; - params[6].is_null = NULL; - - params[7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[7].buffer_length = sizeof(v.f8); - params[7].buffer = &v.f8; - params[7].length = ¶ms[7].buffer_length; - params[7].is_null = NULL; - - params[8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[8].buffer_length = sizeof(v.bin); - params[8].buffer = v.bin; - params[8].length = ¶ms[8].buffer_length; - params[8].is_null = NULL; - - params[9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[9].buffer_length = sizeof(v.bin); - params[9].buffer = v.bin; - params[9].length = ¶ms[9].buffer_length; - params[9].is_null = NULL; - - int is_null = 1; - - char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - for (int zz = 0; zz < 10; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - v.ts = 1591060628000 + zz * 10; - for (int i = 0; i < 10; ++i) { - v.ts += 1; - for (int j = 1; j < 10; ++j) { - params[j].is_null = ((i == j) ? &is_null : 0); - } - v.b = (int8_t)(i+zz*10) % 2; - v.v1 = (int8_t)(i+zz*10); - v.v2 = (int16_t)((i+zz*10) * 2); - v.v4 = (int32_t)((i+zz*10) * 4); - v.v8 = (int64_t)((i+zz*10) * 8); - v.f4 = (float)((i+zz*10) * 40); - v.f8 = (double)((i+zz*10) * 80); - for (int j = 0; j < sizeof(v.bin) - 1; ++j) { - v.bin[j] = (char)((i+zz)%10 + '0'); - } - - taos_stmt_bind_param(stmt, params); - taos_stmt_add_batch(stmt); - } - } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - return 0; -} - - -int stmt_func2(TAOS_STMT *stmt) { - struct { - int64_t ts; - int8_t b; - int8_t v1; - int16_t v2; - int32_t v4; - int64_t v8; - float f4; - double f8; - char bin[40]; - char blob[80]; - } v = {0}; - - TAOS_BIND params[10]; - params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[0].buffer_length = sizeof(v.ts); - params[0].buffer = &v.ts; - params[0].length = ¶ms[0].buffer_length; - params[0].is_null = NULL; - - params[1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[1].buffer_length = sizeof(v.b); - params[1].buffer = &v.b; - params[1].length = ¶ms[1].buffer_length; - params[1].is_null = NULL; - - params[2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[2].buffer_length = sizeof(v.v1); - params[2].buffer = &v.v1; - params[2].length = ¶ms[2].buffer_length; - params[2].is_null = NULL; - - params[3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[3].buffer_length = sizeof(v.v2); - params[3].buffer = &v.v2; - params[3].length = ¶ms[3].buffer_length; - params[3].is_null = NULL; - - params[4].buffer_type = TSDB_DATA_TYPE_INT; - params[4].buffer_length = sizeof(v.v4); - params[4].buffer = &v.v4; - params[4].length = ¶ms[4].buffer_length; - params[4].is_null = NULL; - - params[5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[5].buffer_length = sizeof(v.v8); - params[5].buffer = &v.v8; - params[5].length = ¶ms[5].buffer_length; - params[5].is_null = NULL; - - params[6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[6].buffer_length = sizeof(v.f4); - params[6].buffer = &v.f4; - params[6].length = ¶ms[6].buffer_length; - params[6].is_null = NULL; - - params[7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[7].buffer_length = sizeof(v.f8); - params[7].buffer = &v.f8; - params[7].length = ¶ms[7].buffer_length; - params[7].is_null = NULL; - - params[8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[8].buffer_length = sizeof(v.bin); - params[8].buffer = v.bin; - params[8].length = ¶ms[8].buffer_length; - params[8].is_null = NULL; - - params[9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[9].buffer_length = sizeof(v.bin); - params[9].buffer = v.bin; - params[9].length = ¶ms[9].buffer_length; - params[9].is_null = NULL; - - int is_null = 1; - - char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - for (int l = 0; l < 100; l++) { - for (int zz = 0; zz < 10; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - v.ts = 1591060628000 + zz * 100 * l; - for (int i = 0; i < zz; ++i) { - v.ts += 1; - for (int j = 1; j < 10; ++j) { - params[j].is_null = ((i == j) ? &is_null : 0); - } - v.b = (int8_t)(i+zz*10) % 2; - v.v1 = (int8_t)(i+zz*10); - v.v2 = (int16_t)((i+zz*10) * 2); - v.v4 = (int32_t)((i+zz*10) * 4); - v.v8 = (int64_t)((i+zz*10) * 8); - v.f4 = (float)((i+zz*10) * 40); - v.f8 = (double)((i+zz*10) * 80); - for (int j = 0; j < sizeof(v.bin) - 1; ++j) { - v.bin[j] = (char)((i+zz)%10 + '0'); - } - - taos_stmt_bind_param(stmt, params); - taos_stmt_add_batch(stmt); - } - } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - } - - - return 0; -} - - - - -int stmt_func3(TAOS_STMT *stmt) { - struct { - int64_t ts; - int8_t b; - int8_t v1; - int16_t v2; - int32_t v4; - int64_t v8; - float f4; - double f8; - char bin[40]; - char blob[80]; - } v = {0}; - - TAOS_BIND params[10]; - params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[0].buffer_length = sizeof(v.ts); - params[0].buffer = &v.ts; - params[0].length = ¶ms[0].buffer_length; - params[0].is_null = NULL; - - params[1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[1].buffer_length = sizeof(v.b); - params[1].buffer = &v.b; - params[1].length = ¶ms[1].buffer_length; - params[1].is_null = NULL; - - params[2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[2].buffer_length = sizeof(v.v1); - params[2].buffer = &v.v1; - params[2].length = ¶ms[2].buffer_length; - params[2].is_null = NULL; - - params[3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[3].buffer_length = sizeof(v.v2); - params[3].buffer = &v.v2; - params[3].length = ¶ms[3].buffer_length; - params[3].is_null = NULL; - - params[4].buffer_type = TSDB_DATA_TYPE_INT; - params[4].buffer_length = sizeof(v.v4); - params[4].buffer = &v.v4; - params[4].length = ¶ms[4].buffer_length; - params[4].is_null = NULL; - - params[5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[5].buffer_length = sizeof(v.v8); - params[5].buffer = &v.v8; - params[5].length = ¶ms[5].buffer_length; - params[5].is_null = NULL; - - params[6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[6].buffer_length = sizeof(v.f4); - params[6].buffer = &v.f4; - params[6].length = ¶ms[6].buffer_length; - params[6].is_null = NULL; - - params[7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[7].buffer_length = sizeof(v.f8); - params[7].buffer = &v.f8; - params[7].length = ¶ms[7].buffer_length; - params[7].is_null = NULL; - - params[8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[8].buffer_length = sizeof(v.bin); - params[8].buffer = v.bin; - params[8].length = ¶ms[8].buffer_length; - params[8].is_null = NULL; - - params[9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[9].buffer_length = sizeof(v.bin); - params[9].buffer = v.bin; - params[9].length = ¶ms[9].buffer_length; - params[9].is_null = NULL; - - int is_null = 1; - - char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - for (int l = 0; l < 100; l++) { - for (int zz = 0; zz < 10; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - v.ts = 1591060628000 + zz * 100 * l; - for (int i = 0; i < zz; ++i) { - v.ts += 1; - for (int j = 1; j < 10; ++j) { - params[j].is_null = ((i == j) ? &is_null : 0); - } - v.b = (int8_t)(i+zz*10) % 2; - v.v1 = (int8_t)(i+zz*10); - v.v2 = (int16_t)((i+zz*10) * 2); - v.v4 = (int32_t)((i+zz*10) * 4); - v.v8 = (int64_t)((i+zz*10) * 8); - v.f4 = (float)((i+zz*10) * 40); - v.f8 = (double)((i+zz*10) * 80); - for (int j = 0; j < sizeof(v.bin) - 1; ++j) { - v.bin[j] = (char)((i+zz)%10 + '0'); - } - - taos_stmt_bind_param(stmt, params); - taos_stmt_add_batch(stmt); - } - } - } - - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - - return 0; -} - - //1 tables 10 records int stmt_funcb_autoctb1(TAOS_STMT *stmt) { @@ -4177,13 +3812,10 @@ int stmt_funcb_sc3(TAOS_STMT *stmt) { } #endif -void prepareCheckResultImpl(TAOS *taos, char *tname, int printr, int expected) { +void prepareCheckResultImpl(TAOS *taos, char *tname, bool printr, int expected) { char sql[255] = "SELECT * FROM "; TAOS_RES *result; - //FORCE NO PRINT - //printr = 0; - strcat(sql, tname); result = taos_query(taos, sql); @@ -4232,7 +3864,7 @@ void prepareCheckResult(TAOS *taos) { sprintf(buf, "t%d", 0); } - prepareCheckResultImpl(taos, buf, 1, gCurCase->rowNum * gExecLoopTimes); + prepareCheckResultImpl(taos, buf, gCaseCtrl.printRes, gCurCase->rowNum * gExecLoopTimes); } } @@ -4555,11 +4187,26 @@ void prepare(TAOS *taos, int32_t colNum, int32_t *colList, int autoCreate) { void* runcase(TAOS *taos) { TAOS_STMT *stmt = NULL; + int32_t caseIdx = 0; for (int32_t i = 0; i < sizeof(gCase)/sizeof(gCase[0]); ++i) { - gCurCase = &gCase[i]; - printf("* Case %d - %s Begin *\n", i, gCurCase->caseDesc); + CaseCfg cfg = gCase[i]; + gCurCase = &cfg; + if ((gCaseCtrl.bindColTypeNum || gCaseCtrl.bindColNum) && (gCurCase->colNum != gFullColNum)) { + continue; + } + + printf("* Case %d - %s Begin *\n", caseIdx, gCurCase->caseDesc); + + if (gCaseCtrl.runTimes) { + gCurCase->runTimes = gCaseCtrl.runTimes; + } + + if (gCaseCtrl.rowNum) { + gCurCase->rowNum = gCaseCtrl.rowNum; + } + if (gCurCase->fullCol) { gCurCase->bindColNum = gCurCase->colNum; } @@ -4568,12 +4215,14 @@ void* runcase(TAOS *taos) { gCurCase->autoCreate = gCaseCtrl.autoCreate; if (gCaseCtrl.bindColNum) { gCurCase->bindColNum = gCaseCtrl.bindColNum; + gCurCase->fullCol = false; } if (gCaseCtrl.bindRowNum) { gCurCase->bindRowNum = gCaseCtrl.bindRowNum; } if (gCaseCtrl.bindColTypeNum) { - gCurCase->bindRowNum = gCaseCtrl.bindColTypeNum; + gCurCase->bindColNum = gCaseCtrl.bindColTypeNum; + gCurCase->fullCol = false; } for (int32_t n = 0; n < gCurCase->runTimes; ++n) { @@ -4592,583 +4241,54 @@ void* runcase(TAOS *taos) { taos_stmt_close(stmt); } - printf("* Case %d - %s End *\n", i, gCurCase->caseDesc); + printf("* Case %d - %s End *\n", caseIdx, gCurCase->caseDesc); + + caseIdx++; } - -#if 0 - prepare(taos, 0, 1); - - stmt = taos_stmt_init(taos); - if (NULL == stmt) { - printf("taos_stmt_init failed\n"); - exit(1); - } - - printf("1t+1records start\n"); - stmt_allcol_func1(stmt); - printf("1t+1records end\n"); - printf("check result start\n"); - check_result(taos, "m0", 1, 1); - printf("check result end\n"); - taos_stmt_close(stmt); -#endif - - -#if 0 - prepare(taos, 1, 1); - - stmt = taos_stmt_init(taos); - if (NULL == stmt) { - printf("taos_stmt_init failed\n"); - exit(1); - } - - printf("10t+10records+specifycol start\n"); - stmt_scol_func1(stmt); - printf("10t+10records+specifycol end\n"); - printf("check result start\n"); - check_result(taos, "m0", 1, 10); - check_result(taos, "m1", 1, 10); - check_result(taos, "m2", 1, 10); - check_result(taos, "m3", 1, 10); - check_result(taos, "m4", 1, 10); - check_result(taos, "m5", 1, 10); - check_result(taos, "m6", 1, 10); - check_result(taos, "m7", 1, 10); - check_result(taos, "m8", 1, 10); - check_result(taos, "m9", 1, 10); - printf("check result end\n"); - taos_stmt_close(stmt); -#endif - - - - -#if 0 - prepare(taos, 1, 1); - - stmt = taos_stmt_init(taos); - - printf("1t+100records+specifycol start\n"); - stmt_scol_func2(stmt); - printf("1t+100records+specifycol end\n"); - printf("check result start\n"); - check_result(taos, "m0", 1, 100); - printf("check result end\n"); - taos_stmt_close(stmt); -#endif - - - -#if 0 - prepare(taos, 1, 1); - - stmt = taos_stmt_init(taos); - - printf("300t+10r+bm+specifycol start\n"); - stmt_scol_func3(stmt); - printf("300t+10r+bm+specifycol end\n"); - printf("check result start\n"); - check_result(taos, "m0", 1, 20); - check_result(taos, "m1", 1, 20); - check_result(taos, "m111", 1, 20); - check_result(taos, "m223", 1, 20); - check_result(taos, "m299", 1, 20); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - - -#if 0 - prepare(taos, 1, 1); - - stmt = taos_stmt_init(taos); - - printf("10t+2r+bm+specifycol start\n"); - stmt_scol_func4(stmt); - printf("10t+2r+bm+specifycol end\n"); - printf("check result start\n"); - check_result(taos, "m0", 1, 20); - check_result(taos, "m1", 1, 20); - check_result(taos, "m2", 1, 20); - check_result(taos, "m3", 1, 20); - check_result(taos, "m4", 1, 20); - check_result(taos, "m5", 1, 20); - check_result(taos, "m6", 1, 20); - check_result(taos, "m7", 1, 20); - check_result(taos, "m8", 1, 20); - check_result(taos, "m9", 1, 20); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - -#if 0 - prepare(taos, 1, 1); - - stmt = taos_stmt_init(taos); - - printf("1t+2r+bm+specifycol start\n"); - stmt_scol_func5(stmt); - printf("1t+2r+bm+specifycol end\n"); - printf("check result start\n"); - check_result(taos, "m0", 1, 40); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - - -#if 0 - - -#if 1 - prepare(taos, 1, 1); - - stmt = taos_stmt_init(taos); - - printf("10t+10records start\n"); - stmt_func1(stmt); - printf("10t+10records end\n"); - printf("check result start\n"); - check_result(taos, "m0", 1, 10); - check_result(taos, "m1", 1, 10); - check_result(taos, "m2", 1, 10); - check_result(taos, "m3", 1, 10); - check_result(taos, "m4", 1, 10); - check_result(taos, "m5", 1, 10); - check_result(taos, "m6", 1, 10); - check_result(taos, "m7", 1, 10); - check_result(taos, "m8", 1, 10); - check_result(taos, "m9", 1, 10); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - -#if 1 - prepare(taos, 1, 1); - - stmt = taos_stmt_init(taos); - - printf("10t+[0,1,2...9]records start\n"); - stmt_func2(stmt); - printf("10t+[0,1,2...9]records end\n"); - printf("check result start\n"); - check_result(taos, "m0", 0, 0); - check_result(taos, "m1", 0, 100); - check_result(taos, "m2", 0, 200); - check_result(taos, "m3", 0, 300); - check_result(taos, "m4", 0, 400); - check_result(taos, "m5", 0, 500); - check_result(taos, "m6", 0, 600); - check_result(taos, "m7", 0, 700); - check_result(taos, "m8", 0, 800); - check_result(taos, "m9", 0, 900); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - -#if 1 - prepare(taos, 1, 1); - - stmt = taos_stmt_init(taos); - - printf("10t+[0,100,200...900]records start\n"); - stmt_func3(stmt); - printf("10t+[0,100,200...900]records end\n"); - printf("check result start\n"); - check_result(taos, "m0", 0, 0); - check_result(taos, "m1", 0, 100); - check_result(taos, "m2", 0, 200); - check_result(taos, "m3", 0, 300); - check_result(taos, "m4", 0, 400); - check_result(taos, "m5", 0, 500); - check_result(taos, "m6", 0, 600); - check_result(taos, "m7", 0, 700); - check_result(taos, "m8", 0, 800); - check_result(taos, "m9", 0, 900); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - - -#if 1 - prepare(taos, 1, 1); - - stmt = taos_stmt_init(taos); - - printf("300t+60r+bm start\n"); - stmt_funcb1(stmt); - printf("300t+60r+bm end\n"); - printf("check result start\n"); - check_result(taos, "m0", 0, 180000); - check_result(taos, "m1", 0, 180000); - check_result(taos, "m111", 0, 180000); - check_result(taos, "m223", 0, 180000); - check_result(taos, "m299", 0, 180000); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - -#if 1 - prepare(taos, 1, 0); - - stmt = taos_stmt_init(taos); - - printf("1t+10r+bm+autoctb1 start\n"); - stmt_funcb_autoctb1(stmt); - printf("1t+10r+bm+autoctb1 end\n"); - printf("check result start\n"); - check_result(taos, "m0", 1, 10); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - -#if 1 - prepare(taos, 1, 0); - - stmt = taos_stmt_init(taos); - - printf("1t+10r+bm+autoctb2 start\n"); - stmt_funcb_autoctb2(stmt); - printf("1t+10r+bm+autoctb2 end\n"); - printf("check result start\n"); - check_result(taos, "m0", 1, 10); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - - -#if 1 - prepare(taos, 1, 0); - - stmt = taos_stmt_init(taos); - - printf("1t+10r+bm+autoctb3 start\n"); - stmt_funcb_autoctb3(stmt); - printf("1t+10r+bm+autoctb3 end\n"); - printf("check result start\n"); - check_result(taos, "m0", 1, 10); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - - -#if 1 - prepare(taos, 1, 0); - - stmt = taos_stmt_init(taos); - - printf("1t+10r+bm+autoctb4 start\n"); - stmt_funcb_autoctb4(stmt); - printf("1t+10r+bm+autoctb4 end\n"); - printf("check result start\n"); - check_result(taos, "m0", 1, 10); - printf("check result end\n"); - taos_stmt_close(stmt); -#endif - - -#if 1 - prepare(taos, 1, 0); - - stmt = taos_stmt_init(taos); - - printf("1t+10r+bm+autoctb+e1 start\n"); - stmt_funcb_autoctb_e1(stmt); - printf("1t+10r+bm+autoctb+e1 end\n"); - printf("check result start\n"); - //check_result(taos, "m0", 1, 0); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - -#if 1 - prepare(taos, 1, 0); - - stmt = taos_stmt_init(taos); - - printf("1t+10r+bm+autoctb+e2 start\n"); - stmt_funcb_autoctb_e2(stmt); - printf("1t+10r+bm+autoctb+e2 end\n"); - printf("check result start\n"); - //check_result(taos, "m0", 1, 0); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - -#if 1 - prepare(taos, 1, 0); - - stmt = taos_stmt_init(taos); - - printf("1t+10r+bm+autoctb+e3 start\n"); - stmt_funcb_autoctb_e3(stmt); - printf("1t+10r+bm+autoctb+e3 end\n"); - printf("check result start\n"); - //check_result(taos, "m0", 1, 0); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - - -#if 1 - prepare(taos, 1, 0); - - stmt = taos_stmt_init(taos); - - printf("1t+10r+bm+autoctb+e4 start\n"); - stmt_funcb_autoctb_e4(stmt); - printf("1t+10r+bm+autoctb+e4 end\n"); - printf("check result start\n"); - //check_result(taos, "m0", 1, 0); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - -#if 1 - prepare(taos, 1, 0); - - stmt = taos_stmt_init(taos); - - printf("1t+10r+bm+autoctb+e5 start\n"); - stmt_funcb_autoctb_e5(stmt); - printf("1t+10r+bm+autoctb+e5 end\n"); - printf("check result start\n"); - //check_result(taos, "m0", 1, 0); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - - -#if 1 - prepare(taos, 1, 1); - - stmt = taos_stmt_init(taos); - - printf("1t+18000r+bm start\n"); - stmt_funcb2(stmt); - printf("1t+18000r+bm end\n"); - printf("check result start\n"); - check_result(taos, "m0", 0, 180000); - check_result(taos, "m1", 0, 180000); - check_result(taos, "m111", 0, 180000); - check_result(taos, "m223", 0, 180000); - check_result(taos, "m299", 0, 180000); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - -#if 1 - prepare(taos, 1, 1); - - stmt = taos_stmt_init(taos); - - printf("300t+60r+disorder+bm start\n"); - stmt_funcb3(stmt); - printf("300t+60r+disorder+bm end\n"); - printf("check result start\n"); - check_result(taos, "m0", 0, 180000); - check_result(taos, "m1", 0, 180000); - check_result(taos, "m111", 0, 180000); - check_result(taos, "m223", 0, 180000); - check_result(taos, "m299", 0, 180000); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - - -#if 1 - prepare(taos, 1, 1); - - stmt = taos_stmt_init(taos); - - printf("300t+60r+samets+bm start\n"); - stmt_funcb4(stmt); - printf("300t+60r+samets+bm end\n"); - printf("check result start\n"); - check_result(taos, "m0", 0, 1); - check_result(taos, "m1", 0, 1); - check_result(taos, "m111", 0, 1); - check_result(taos, "m223", 0, 1); - check_result(taos, "m299", 0, 1); - printf("check result end\n"); - taos_stmt_close(stmt); -#endif - -#if 1 - prepare(taos, 1, 1); - - stmt = taos_stmt_init(taos); - - printf("1t+18000r+nodyntable+bm start\n"); - stmt_funcb5(stmt); - printf("1t+18000r+nodyntable+bm end\n"); - printf("check result start\n"); - check_result(taos, "m0", 0, 180000); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - - -#if 1 - prepare(taos, 1, 1); - - stmt = taos_stmt_init(taos); - - printf("300t+60r+bm+sc start\n"); - stmt_funcb_sc1(stmt); - printf("300t+60r+bm+sc end\n"); - printf("check result start\n"); - check_result(taos, "m0", 0, 180000); - check_result(taos, "m1", 0, 180000); - check_result(taos, "m111", 0, 180000); - check_result(taos, "m223", 0, 180000); - check_result(taos, "m299", 0, 180000); - printf("check result end\n"); - taos_stmt_close(stmt); -#endif - -#if 1 - prepare(taos, 1, 1); - - stmt = taos_stmt_init(taos); - - printf("1t+60r+bm+sc start\n"); - stmt_funcb_sc2(stmt); - printf("1t+60r+bm+sc end\n"); - printf("check result start\n"); - check_result(taos, "m0", 0, 180000); - check_result(taos, "m1", 0, 180000); - check_result(taos, "m111", 0, 180000); - check_result(taos, "m223", 0, 180000); - check_result(taos, "m299", 0, 180000); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - -#if 1 - prepare(taos, 1, 1); - - stmt = taos_stmt_init(taos); - - printf("10t+[1...10]r+bm+sc start\n"); - stmt_funcb_sc3(stmt); - printf("10t+[1...10]r+bm+sc end\n"); - printf("check result start\n"); - check_result(taos, "m0", 1, 1); - check_result(taos, "m1", 1, 2); - check_result(taos, "m2", 1, 3); - check_result(taos, "m3", 1, 4); - check_result(taos, "m4", 1, 5); - check_result(taos, "m5", 1, 6); - check_result(taos, "m6", 1, 7); - check_result(taos, "m7", 1, 8); - check_result(taos, "m8", 1, 9); - check_result(taos, "m9", 1, 10); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - - -#if 1 - prepare(taos, 1, 1); - - stmt = taos_stmt_init(taos); - - printf("1t+60r+bm start\n"); - stmt_funcb_s1(stmt); - printf("1t+60r+bm end\n"); - printf("check result start\n"); - check_result(taos, "m0", 0, 180000); - check_result(taos, "m1", 0, 180000); - check_result(taos, "m111", 0, 180000); - check_result(taos, "m223", 0, 180000); - check_result(taos, "m299", 0, 180000); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif - - -#if 1 - prepare(taos, 1, 1); - - (void)stmt; - printf("120t+60r+sql start\n"); - sql_perf1(taos); - printf("120t+60r+sql end\n"); - printf("check result start\n"); - check_result(taos, "m0", 0, 180000); - check_result(taos, "m1", 0, 180000); - check_result(taos, "m34", 0, 180000); - check_result(taos, "m67", 0, 180000); - check_result(taos, "m99", 0, 180000); - printf("check result end\n"); -#endif - -#if 1 - prepare(taos, 1, 1); - - (void)stmt; - printf("1t+60r+sql start\n"); - sql_perf_s1(taos); - printf("1t+60r+sql end\n"); - printf("check result start\n"); - check_result(taos, "m0", 0, 180000); - check_result(taos, "m1", 0, 180000); - check_result(taos, "m34", 0, 180000); - check_result(taos, "m67", 0, 180000); - check_result(taos, "m99", 0, 180000); - printf("check result end\n"); -#endif - - -#if 1 - preparem(taos, 0, idx); - - stmt = taos_stmt_init(taos); - - printf("1t+30000r+bm start\n"); - stmt_funcb_ssz1(stmt); - printf("1t+30000r+bm end\n"); - printf("check result start\n"); - check_result(taos, "m0", 0, 300000); - check_result(taos, "m1", 0, 300000); - check_result(taos, "m111", 0, 300000); - check_result(taos, "m223", 0, 300000); - check_result(taos, "m299", 0, 300000); - printf("check result end\n"); - taos_stmt_close(stmt); - -#endif -#endif - printf("test end\n"); return NULL; } +void runAll(TAOS *taos) { +#if 0 + runcase(taos); + + gCaseCtrl.bindNullNum = 1; + runcase(taos); + gCaseCtrl.bindNullNum = 0; + + gCaseCtrl.bindRowNum = 1; + runcase(taos); + gCaseCtrl.bindRowNum = 0; +#endif + gCaseCtrl.rowNum = 10000; + gCaseCtrl.printRes = false; + runcase(taos); + gCaseCtrl.rowNum = 0; + gCaseCtrl.printRes = true; +#if 0 + + gCaseCtrl.runTimes = 10; + runcase(taos); + gCaseCtrl.runTimes = 0; + + gCaseCtrl.checkParamNum = true; + runcase(taos); + gCaseCtrl.checkParamNum = false; + + gCaseCtrl.bindColNum = 6; + runcase(taos); + gCaseCtrl.bindColNum = 0; + + gCaseCtrl.bindColTypeNum = tListLen(bindColTypeList); + gCaseCtrl.bindColTypeNum = bindColTypeList = bindColTypeList; + runcase(taos); +#endif +} + int main(int argc, char *argv[]) { TAOS *taos = NULL; @@ -5187,7 +4307,7 @@ int main(int argc, char *argv[]) exit(1); } - runcase(taos); + runAll(taos); return 0; } From 3ecf6df9e362ba5b18a54985048b6462775a7fb0 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Wed, 20 Apr 2022 20:29:50 +0800 Subject: [PATCH 40/47] [test: add valgrind case] --- tests/script/jenkins/basic.txt | 3 +++ tests/script/tsim/valgrind/checkError.sim | 11 +++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 64ea437648..506abf3fc5 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -93,4 +93,7 @@ # --- sma ./test.sh -f tsim/sma/tsmaCreateInsertData.sim +# --- valgrind +./test.sh -f tsim/valgrind/checkError.sim -v + #======================b1-end=============== diff --git a/tests/script/tsim/valgrind/checkError.sim b/tests/script/tsim/valgrind/checkError.sim index 3b66cb4a8c..80700dcbce 100644 --- a/tests/script/tsim/valgrind/checkError.sim +++ b/tests/script/tsim/valgrind/checkError.sim @@ -71,6 +71,13 @@ print ====> start to check if there are ERRORS in vagrind log file for each dnod # -n : dnode[x] be check system_content sh/checkValgrind.sh -n dnode1 print cmd return result----> [ $system_content ] -if $system_content != @ERROR SUMMARY: 0 errors@ then - return -1 +if $system_content == @ERROR SUMMARY: 0 errors@ then + return 0 endi + +$null= +if $system_content == $null then + return 0 +endi + +return -1 \ No newline at end of file From 29d994d1dd5040f1585008207dc26e1ab57006c4 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 20 Apr 2022 20:34:45 +0800 Subject: [PATCH 41/47] feat: taos shell refactor (#11688) * [TD-13558]: taos shell refactor add taosTools as submodule * add tools/taos-tools * add more client interface for taosTools compile * update taos-tools * update taos-tools * refactor shell * [TD-13558]: taos shell test speed * [TD-13558]: taos -n startup works * taos -n rpc works * taos -n server works * cleanup code since no endPort in 3.0 * update taos-tools * [TD-13558]: taos -C works * improve taos shell -c WIP * update taos-tools * add demoapi.c * adjust show databases result for 3.0 * test: add platform logic * add nchar * adjust taos_fetch_lengths * print fields * remove show databases check from insert cases * fix lua example compile for 3.0 still not work * remove lua.py from smoketest * use get_column_data_offset() to get offset and convert length * add int, float, timestamp value convert * improve tests/pytest/insert/binary.py * improve example/src/demoapi.c --- example/src/demoapi.c | 2 +- tests/pytest/insert/binary.py | 28 +++++++++++++++------------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/example/src/demoapi.c b/example/src/demoapi.c index 45da88fa1b..afa9e19d4d 100644 --- a/example/src/demoapi.c +++ b/example/src/demoapi.c @@ -176,7 +176,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { } if (block) { - warnPrint("%s", "call taos_fetch_block()\n"); + warnPrint("%s", "call taos_fetch_block(), don't call taos_fetch_lengths()\n"); int rows = 0; while ((rows = taos_fetch_block(res, &row))) { int *lengths = taos_fetch_lengths(res); diff --git a/tests/pytest/insert/binary.py b/tests/pytest/insert/binary.py index ffd1d6cb8c..28621f777b 100644 --- a/tests/pytest/insert/binary.py +++ b/tests/pytest/insert/binary.py @@ -35,8 +35,8 @@ class TDTestCase: tdSql.prepare() tdLog.info('=============== step1') - tdLog.info('create table tb (ts timestamp, speed binary(5))') - tdSql.execute('create table tb (ts timestamp, speed binary(5))') + tdLog.info('create table tb (ts timestamp, speed binary(10))') + tdSql.execute('create table tb (ts timestamp, speed binary(10))') tdLog.info("insert into tb values (now, ) -x step1") tdSql.error("insert into tb values (now, )") tdLog.info('=============== step2') @@ -49,27 +49,29 @@ class TDTestCase: tdLog.info("tdSql.checkData(0, 0, '1234')") tdSql.checkData(0, 0, '1234') tdLog.info('=============== step3') - tdLog.info("insert into tb values (now+2a, '23456')") - tdSql.execute("insert into tb values (now+2a, '23456')") + tdLog.info("insert into tb values (now+2a, '0123456789')") + tdSql.execute("insert into tb values (now+2a, '0123456789')") tdLog.info('select speed from tb order by ts desc') tdSql.query('select speed from tb order by ts desc') tdLog.info('tdSql.checkRow(2)') tdSql.checkRows(2) tdLog.info('==> $data00') - tdLog.info("tdSql.checkData(0, 0, '23456')") - tdSql.checkData(0, 0, '23456') + tdLog.info("tdSql.checkData(0, 0, '0123456789')") + tdSql.checkData(0, 0, '0123456789') tdLog.info('=============== step4') - tdLog.info("insert into tb values (now+3a, '345678')") - tdSql.error("insert into tb values (now+3a, '345678')") + tdLog.info("insert into tb values (now+3a, '01234567890')") + tdSql.error("insert into tb values (now+3a, '01234567890')") tdLog.info("insert into tb values (now+3a, '34567')") tdSql.execute("insert into tb values (now+3a, '34567')") + tdLog.info("insert into tb values (now+4a, NULL)") + tdSql.execute("insert into tb values (now+4a, NULL)") tdLog.info('select speed from tb order by ts desc') tdSql.query('select speed from tb order by ts desc') - tdLog.info('tdSql.checkRow(3)') - tdSql.checkRows(3) - tdLog.info('==> $data00') - tdLog.info("tdSql.checkData(0, 0, '34567')") - tdSql.checkData(0, 0, '34567') + tdSql.checkRows(4) + tdLog.info("tdSql.checkData(0, 0, '0123456789')") + tdSql.checkData(0, 0, '0123456789') + tdLog.info("tdSql.checkData(3, 0, None)") + tdSql.checkData(3, 0, None) tdLog.info("insert into tb values (now+4a, \"'';\")") if platform.system() == "Linux": From d75ab9b3ff053db2ea017bd1c1fdb5b70f4037a1 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 20 Apr 2022 21:36:55 +0800 Subject: [PATCH 42/47] refactor --- example/src/tmq.c | 3 +- include/common/tmsg.h | 79 +++++- source/client/inc/clientInt.h | 60 ++--- source/client/src/clientMain.c | 35 ++- source/client/src/tmq.c | 210 ++++++---------- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 1 + source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 11 +- source/dnode/mnode/impl/src/mndConsumer.c | 15 +- source/dnode/mnode/impl/src/mndDef.c | 29 ++- source/dnode/mnode/impl/src/mndScheduler.c | 17 +- source/dnode/mnode/impl/src/mndSubscribe.c | 74 +++--- source/dnode/mnode/impl/src/mnode.c | 6 +- source/dnode/vnode/src/inc/tq.h | 49 ++-- source/dnode/vnode/src/tq/tq.c | 251 +++++++++----------- source/dnode/vnode/src/tq/tqMetaStore.c | 1 - source/dnode/vnode/src/vnd/vnodeSvr.c | 10 +- source/os/src/osFile.c | 40 ++-- source/util/src/tarray.c | 7 +- tests/script/tsim/tmq/basic1.sim | 4 +- tests/test/c/tmqSim.c | 6 +- 20 files changed, 480 insertions(+), 428 deletions(-) diff --git a/example/src/tmq.c b/example/src/tmq.c index fdd26bc95d..4847e243a1 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -22,6 +22,7 @@ static int running = 1; static void msg_process(TAOS_RES* msg) { char buf[1024]; + memset(buf, 0, 1024); printf("topic: %s\n", tmq_get_topic_name(msg)); printf("vg:%d\n", tmq_get_vgroup_id(msg)); while (1) { @@ -220,7 +221,7 @@ void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { msg_process(tmqmessage); tmq_message_destroy(tmqmessage); - if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0); + /*if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0);*/ } } diff --git a/include/common/tmsg.h b/include/common/tmsg.h index ecb5cd3d61..2b480a4067 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2396,6 +2396,7 @@ typedef struct { int64_t consumerId; } SMqRspHead; +#if 0 typedef struct { SMsgHead head; @@ -2409,6 +2410,17 @@ typedef struct { uint64_t reqId; char topic[TSDB_TOPIC_FNAME_LEN]; } SMqPollReq; +#endif + +typedef struct { + SMsgHead head; + char subKey[TSDB_SUBSCRIBE_KEY_LEN]; + int32_t epoch; + uint64_t reqId; + int64_t consumerId; + int64_t blockingTime; + int64_t currentOffset; +} SMqPollReqV2; typedef struct { int32_t vgId; @@ -2482,6 +2494,71 @@ static FORCE_INLINE void* tDecodeSMqPollRspV2(const void* buf, SMqPollRspV2* pRs return (void*)buf; } +typedef struct { + SMqRspHead head; + int64_t reqOffset; + int64_t rspOffset; + int32_t skipLogNum; + int32_t blockNum; + int8_t withTbName; + int8_t withSchema; + int8_t withTag; + int8_t withTagSchema; + SArray* blockDataLen; // SArray + SArray* blockData; // SArray + SArray* blockTbName; // SArray + SArray* blockSchema; // SArray + SArray* blockTags; // SArray + SArray* blockTagSchema; // SArray +} SMqDataBlkRsp; + +static FORCE_INLINE int32_t tEncodeSMqDataBlkRsp(void** buf, const SMqDataBlkRsp* pRsp) { + int32_t tlen = 0; + tlen += taosEncodeFixedI64(buf, pRsp->reqOffset); + tlen += taosEncodeFixedI64(buf, pRsp->rspOffset); + tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum); + tlen += taosEncodeFixedI32(buf, pRsp->blockNum); + if (pRsp->blockNum != 0) { + tlen += taosEncodeFixedI8(buf, pRsp->withTbName); + tlen += taosEncodeFixedI8(buf, pRsp->withSchema); + tlen += taosEncodeFixedI8(buf, pRsp->withTag); + tlen += taosEncodeFixedI8(buf, pRsp->withTagSchema); + + for (int32_t i = 0; i < pRsp->blockNum; i++) { + int32_t bLen = *(int32_t*)taosArrayGet(pRsp->blockDataLen, i); + void* data = taosArrayGetP(pRsp->blockData, i); + tlen += taosEncodeFixedI32(buf, bLen); + tlen += taosEncodeBinary(buf, data, bLen); + } + } + return tlen; +} + +static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* pRsp) { + buf = taosDecodeFixedI64(buf, &pRsp->reqOffset); + buf = taosDecodeFixedI64(buf, &pRsp->rspOffset); + buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum); + buf = taosDecodeFixedI32(buf, &pRsp->blockNum); + pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void*)); + pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(void*)); + if (pRsp->blockNum != 0) { + buf = taosDecodeFixedI8(buf, &pRsp->withTbName); + buf = taosDecodeFixedI8(buf, &pRsp->withSchema); + buf = taosDecodeFixedI8(buf, &pRsp->withTag); + buf = taosDecodeFixedI8(buf, &pRsp->withTagSchema); + + for (int32_t i = 0; i < pRsp->blockNum; i++) { + int32_t bLen = 0; + void* data = NULL; + buf = taosDecodeFixedI32(buf, &bLen); + buf = taosDecodeBinary(buf, &data, bLen); + taosArrayPush(pRsp->blockDataLen, &bLen); + taosArrayPush(pRsp->blockData, &data); + } + } + return (void*)buf; +} + typedef struct { SMqRspHead head; char cgroup[TSDB_CGROUP_LEN]; @@ -2489,7 +2566,7 @@ typedef struct { } SMqCMGetSubEpRsp; static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { - taosMemoryFree(pSubTopicEp->schema.pSchema); + // taosMemoryFree(pSubTopicEp->schema.pSchema); taosArrayDestroy(pSubTopicEp->vgs); } diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 772ff5e69a..9363cf00a2 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -44,7 +44,7 @@ extern "C" { } while (0) #define ERROR_MSG_BUF_DEFAULT_SIZE 512 -#define HEARTBEAT_INTERVAL 1500 // ms +#define HEARTBEAT_INTERVAL 1500 // ms enum { RES_TYPE__QUERY = 1, @@ -187,11 +187,13 @@ typedef struct SRequestSendRecvBody { } SRequestSendRecvBody; typedef struct { - int8_t resType; - char* topic; - SArray* res; // SArray - int32_t resIter; - int32_t vgId; + int8_t resType; + char topic[TSDB_TOPIC_FNAME_LEN]; + int32_t vgId; + SSchemaWrapper schema; + int32_t resIter; + SMqDataBlkRsp rsp; + SReqResultInfo resInfo; } SMqRspObj; typedef struct SRequestObj { @@ -211,16 +213,24 @@ typedef struct SRequestObj { SRequestSendRecvBody body; } SRequestObj; +void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4); +void doSetOneRowPtr(SReqResultInfo* pResultInfo); +void setResPrecision(SReqResultInfo* pResInfo, int32_t precision); +int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4); +void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols); + static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) { SMqRspObj* msg = (SMqRspObj*)res; - int32_t resIter = msg->resIter == -1 ? 0 : msg->resIter; - return (SReqResultInfo*)taosArrayGet(msg->res, resIter); + return (SReqResultInfo*)&msg->resInfo; } -static FORCE_INLINE SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res) { +static FORCE_INLINE SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) { SMqRspObj* msg = (SMqRspObj*)res; - if (++msg->resIter < taosArrayGetSize(msg->res)) { - return (SReqResultInfo*)taosArrayGet(msg->res, msg->resIter); + msg->resIter++; + if (msg->resIter < msg->rsp.blockNum) { + SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(msg->rsp.blockData, msg->resIter); + setQueryResultFromRsp(&msg->resInfo, pRetrieve, convertUcs4); + return &msg->resInfo; } return NULL; } @@ -238,25 +248,25 @@ extern int (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t int genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code); SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pReqObj); -int taos_init(); +int taos_init(); -void* createTscObj(const char* user, const char* auth, const char* db, SAppInstInfo* pAppInfo); -void destroyTscObj(void* pObj); -STscObj *acquireTscObj(int64_t rid); -int32_t releaseTscObj(int64_t rid); +void* createTscObj(const char* user, const char* auth, const char* db, SAppInstInfo* pAppInfo); +void destroyTscObj(void* pObj); +STscObj* acquireTscObj(int64_t rid); +int32_t releaseTscObj(int64_t rid); uint64_t generateRequestId(); -void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type); -void destroyRequest(SRequestObj* pRequest); -SRequestObj *acquireRequest(int64_t rid); -int32_t releaseRequest(int64_t rid); +void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type); +void destroyRequest(SRequestObj* pRequest); +SRequestObj* acquireRequest(int64_t rid); +int32_t releaseRequest(int64_t rid); char* getDbOfConnection(STscObj* pObj); void setConnectionDB(STscObj* pTscObj, const char* db); void resetConnectDB(STscObj* pTscObj); -int taos_options_imp(TSDB_OPTION option, const char* str); +int taos_options_imp(TSDB_OPTION option, const char* str); void* openTransporter(const char* user, const char* auth, int32_t numOfThreads); @@ -273,12 +283,6 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest); -void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4); -void doSetOneRowPtr(SReqResultInfo* pResultInfo); -void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols); -void setResPrecision(SReqResultInfo* pResInfo, int32_t precision); -int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4); - // --- heartbeat // global, called by mgmt int hbMgrInit(); @@ -290,7 +294,7 @@ SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char* key); void appHbMgrCleanup(void); // conn level -int hbRegisterConn(SAppHbMgr *pAppHbMgr, int64_t tscRefId, int64_t clusterId, int8_t connType); +int hbRegisterConn(SAppHbMgr* pAppHbMgr, int64_t tscRefId, int64_t clusterId, int8_t connType); void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey); int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* value, int32_t keyLen, int32_t valueLen); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 903018d5c3..6472dea556 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -14,12 +14,12 @@ */ #include "catalog.h" -#include "scheduler.h" #include "clientInt.h" -#include "clientStmt.h" #include "clientLog.h" +#include "clientStmt.h" #include "os.h" #include "query.h" +#include "scheduler.h" #include "tglobal.h" #include "tmsg.h" #include "tref.h" @@ -177,25 +177,24 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { return doFetchRows(pRequest, true, true); } else if (TD_RES_TMQ(res)) { - SMqRspObj *msg = ((SMqRspObj *)res); - if (msg->resIter == -1) msg->resIter++; - SReqResultInfo *pResultInfo = taosArrayGet(msg->res, msg->resIter); + SMqRspObj *msg = ((SMqRspObj *)res); + SReqResultInfo *pResultInfo; + if (msg->resIter == -1) { + pResultInfo = tmqGetNextResInfo(res, true); + } else { + pResultInfo = tmqGetCurResInfo(res); + } if (pResultInfo->current < pResultInfo->numOfRows) { doSetOneRowPtr(pResultInfo); pResultInfo->current += 1; return pResultInfo->row; } else { - msg->resIter++; - if (msg->resIter < taosArrayGetSize(msg->res)) { - pResultInfo = taosArrayGet(msg->res, msg->resIter); - doSetOneRowPtr(pResultInfo); - pResultInfo->current += 1; - return pResultInfo->row; - } else { - return NULL; - } + pResultInfo = tmqGetNextResInfo(res, true); + if (pResultInfo == NULL) return NULL; + doSetOneRowPtr(pResultInfo); + pResultInfo->current += 1; + return pResultInfo->row; } - } else { // assert to avoid un-initialization error ASSERT(0); @@ -455,7 +454,7 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) { (*numOfRows) = pResultInfo->numOfRows; return pRequest->code; } else if (TD_RES_TMQ(res)) { - SReqResultInfo *pResultInfo = tmqGetNextResInfo(res); + SReqResultInfo *pResultInfo = tmqGetNextResInfo(res, true); if (pResultInfo == NULL) return -1; pResultInfo->current = pResultInfo->numOfRows; @@ -474,7 +473,7 @@ int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) { } if (TD_RES_TMQ(res)) { - SReqResultInfo *pResultInfo = tmqGetNextResInfo(res); + SReqResultInfo *pResultInfo = tmqGetNextResInfo(res, false); if (pResultInfo == NULL) { (*numOfRows) = 0; return 0; @@ -710,10 +709,8 @@ int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { return stmtBindBatch(stmt, bind); } - TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision) { // TODO return NULL; } - diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 9ea7a85d9b..90a589c36c 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -72,25 +72,25 @@ struct tmq_conf_t { struct tmq_t { // conf - char groupId[TSDB_CGROUP_LEN]; - char clientId[256]; - int8_t autoCommit; - int8_t inWaiting; + char groupId[TSDB_CGROUP_LEN]; + char clientId[256]; + int8_t autoCommit; + /*int8_t inWaiting;*/ int64_t consumerId; int32_t epoch; int32_t resetOffsetCfg; int64_t status; STscObj* pTscObj; tmq_commit_cb* commit_cb; - int32_t nextTopicIdx; - int8_t epStatus; - int32_t epSkipCnt; - int32_t waitingRequest; - int32_t readyRequest; - SArray* clientTopics; // SArray - STaosQueue* mqueue; // queue of tmq_message_t - STaosQall* qall; - tsem_t rspSem; + /*int32_t nextTopicIdx;*/ + int8_t epStatus; + int32_t epSkipCnt; + /*int32_t waitingRequest;*/ + /*int32_t readyRequest;*/ + SArray* clientTopics; // SArray + STaosQueue* mqueue; // queue of tmq_message_t + STaosQall* qall; + tsem_t rspSem; // stat int64_t pollCnt; }; @@ -134,7 +134,7 @@ typedef struct { int32_t epoch; SMqClientVg* vgHandle; SMqClientTopic* topicHandle; - SMqPollRspV2 msg; + SMqDataBlkRsp msg; } SMqPollRspWrapper; typedef struct { @@ -145,6 +145,7 @@ typedef struct { typedef struct { tmq_t* tmq; + int32_t code; int32_t sync; tsem_t rspSem; } SMqAskEpCbParam; @@ -327,12 +328,12 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs return NULL; } pTmq->pTscObj = (STscObj*)conn; - pTmq->inWaiting = 0; + /*pTmq->inWaiting = 0;*/ pTmq->status = 0; pTmq->pollCnt = 0; pTmq->epoch = 0; - pTmq->waitingRequest = 0; - pTmq->readyRequest = 0; + /*pTmq->waitingRequest = 0;*/ + /*pTmq->readyRequest = 0;*/ pTmq->epStatus = 0; pTmq->epSkipCnt = 0; // set conf @@ -372,12 +373,12 @@ tmq_t* tmq_consumer_new1(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->pTscObj = taos_connect_internal(conf->ip, user, pass, NULL, conf->db, conf->port, CONN_TYPE__TMQ); if (pTmq->pTscObj == NULL) return NULL; - pTmq->inWaiting = 0; + /*pTmq->inWaiting = 0;*/ pTmq->status = 0; pTmq->pollCnt = 0; pTmq->epoch = 0; - pTmq->waitingRequest = 0; - pTmq->readyRequest = 0; + /*pTmq->waitingRequest = 0;*/ + /*pTmq->readyRequest = 0;*/ pTmq->epStatus = 0; pTmq->epSkipCnt = 0; // set conf @@ -862,7 +863,6 @@ void tmqShowMsg(tmq_message_t* tmq_message) { #endif int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { - /*printf("recv poll\n");*/ SMqPollCbParam* pParam = (SMqPollCbParam*)param; SMqClientVg* pVg = pParam->pVg; SMqClientTopic* pTopic = pParam->pTopic; @@ -875,17 +875,15 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { int32_t msgEpoch = ((SMqRspHead*)pMsg->pData)->epoch; int32_t tmqEpoch = atomic_load_32(&tmq->epoch); if (msgEpoch < tmqEpoch) { - /*printf("discard rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch);*/ - /*tsem_post(&tmq->rspSem);*/ + // do not write into queue since updating epoch reset tscWarn("msg discard from vg %d since from earlier epoch, rsp epoch %d, current epoch %d", pParam->vgId, msgEpoch, tmqEpoch); + /*tsem_post(&tmq->rspSem);*/ return 0; } if (msgEpoch != tmqEpoch) { tscWarn("mismatch rsp from vg %d, epoch %d, current epoch %d", pParam->vgId, msgEpoch, tmqEpoch); - } else { - atomic_sub_fetch_32(&tmq->waitingRequest, 1); } #if 0 @@ -907,45 +905,33 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { } #endif - /*SMqConsumeRsp* pRsp = taosMemoryCalloc(1, sizeof(SMqConsumeRsp));*/ - /*tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t));*/ SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper)); if (pRspWrapper == NULL) { tscWarn("msg discard from vg %d, epoch %d since out of memory", pParam->vgId, pParam->epoch); goto CREATE_MSG_FAIL; } + pRspWrapper->tmqRspType = TMQ_MSG_TYPE__POLL_RSP; pRspWrapper->vgHandle = pVg; pRspWrapper->topicHandle = pTopic; - /*memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead));*/ - memcpy(&pRspWrapper->msg, pMsg->pData, sizeof(SMqRspHead)); - tDecodeSMqPollRspV2(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRspWrapper->msg); - // TODO: alloc mem - /*pRsp->*/ - /*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/ -#if 0 - if (pRsp->msg.numOfTopics == 0) { - /*printf("no data\n");*/ - taosFreeQitem(pRsp); - goto CREATE_MSG_FAIL; - } -#endif + memcpy(&pRspWrapper->msg, pMsg->pData, sizeof(SMqRspHead)); + + tDecodeSMqDataBlkRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRspWrapper->msg); tscDebug("consumer %ld recv poll: vg %d, req offset %ld, rsp offset %ld", tmq->consumerId, pVg->vgId, pRspWrapper->msg.reqOffset, pRspWrapper->msg.rspOffset); taosWriteQitem(tmq->mqueue, pRspWrapper); - atomic_add_fetch_32(&tmq->readyRequest, 1); /*tsem_post(&tmq->rspSem);*/ - return 0; + return 0; CREATE_MSG_FAIL: if (pParam->epoch == tmq->epoch) { atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); } /*tsem_post(&tmq->rspSem);*/ - return code; + return -1; } bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { @@ -1028,6 +1014,7 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; tmq_t* tmq = pParam->tmq; + pParam->code = code; if (code != 0) { tscError("consumer %ld get topic endpoint error, not ready, wait:%d", tmq->consumerId, pParam->sync); goto END; @@ -1067,6 +1054,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { taosWriteQitem(tmq->mqueue, pWrapper); /*tsem_post(&tmq->rspSem);*/ + taosMemoryFree(pParam); } END: @@ -1078,7 +1066,8 @@ END: } int32_t tmqAskEp(tmq_t* tmq, bool sync) { - int8_t epStatus = atomic_val_compare_exchange_8(&tmq->epStatus, 0, 1); + int32_t code = 0; + int8_t epStatus = atomic_val_compare_exchange_8(&tmq->epStatus, 0, 1); if (epStatus == 1) { int32_t epSkipCnt = atomic_add_fetch_32(&tmq->epSkipCnt, 1); tscTrace("consumer %ld skip ask ep cnt %d", tmq->consumerId, epSkipCnt); @@ -1135,8 +1124,12 @@ int32_t tmqAskEp(tmq_t* tmq, bool sync) { int64_t transporterId = 0; asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); - if (sync) tsem_wait(&pParam->rspSem); - return 0; + if (sync) { + tsem_wait(&pParam->rspSem); + code = pParam->code; + taosMemoryFree(pParam); + } + return code; } tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) { @@ -1162,7 +1155,7 @@ tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) { return TMQ_RESP_ERR__FAIL; } -SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTopic* pTopic, SMqClientVg* pVg) { +SMqPollReqV2* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTopic* pTopic, SMqClientVg* pVg) { int64_t reqOffset; if (pVg->currentOffset >= 0) { reqOffset = pVg->currentOffset; @@ -1174,13 +1167,18 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTo reqOffset = tmq->resetOffsetCfg; } - SMqPollReq* pReq = taosMemoryMalloc(sizeof(SMqPollReq)); + SMqPollReqV2* pReq = taosMemoryMalloc(sizeof(SMqPollReqV2)); if (pReq == NULL) { return NULL; } - strcpy(pReq->topic, pTopic->topicName); - strcpy(pReq->cgroup, tmq->groupId); + /*strcpy(pReq->topic, pTopic->topicName);*/ + /*strcpy(pReq->cgroup, tmq->groupId);*/ + + int32_t tlen = strlen(tmq->groupId); + memcpy(pReq->subKey, tmq->groupId, tlen); + pReq->subKey[tlen] = TMQ_SEPARATOR; + strcpy(pReq->subKey + tlen + 1, pTopic->topicName); pReq->blockingTime = blockingTime; pReq->consumerId = tmq->consumerId; @@ -1189,102 +1187,27 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTo pReq->reqId = generateRequestId(); pReq->head.vgId = htonl(pVg->vgId); - pReq->head.contLen = htonl(sizeof(SMqPollReq)); + pReq->head.contLen = htonl(sizeof(SMqPollReqV2)); return pReq; } SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) { SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj)); pRspObj->resType = RES_TYPE__TMQ; - pRspObj->topic = strdup(pWrapper->topicHandle->topicName); - pRspObj->resIter = -1; + strncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); pRspObj->vgId = pWrapper->vgHandle->vgId; - SMqPollRspV2* pRsp = &pWrapper->msg; - int32_t blockNum = taosArrayGetSize(pRsp->blockPos); - pRspObj->res = taosArrayInit(blockNum, sizeof(SReqResultInfo)); - for (int32_t i = 0; i < blockNum; i++) { - int32_t pos = *(int32_t*)taosArrayGet(pRsp->blockPos, i); - SRetrieveTableRsp* pRetrieve = POINTER_SHIFT(pRsp->blockData, pos); - SReqResultInfo resInfo = {0}; - resInfo.totalRows = 0; - resInfo.precision = TSDB_TIME_PRECISION_MILLI; - setResSchemaInfo(&resInfo, pWrapper->topicHandle->schema.pSchema, pWrapper->topicHandle->schema.nCols); - setQueryResultFromRsp(&resInfo, pRetrieve, true); - taosArrayPush(pRspObj->res, &resInfo); - } + pRspObj->resIter = -1; + memcpy(&pRspObj->rsp, &pWrapper->msg, sizeof(SMqDataBlkRsp)); + + /*SRetrieveTableRsp* pRetrieve = taosArrayGetP(pWrapper->msg.blockData, 0);*/ + pRspObj->resInfo.totalRows = 0; + pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI; + setResSchemaInfo(&pRspObj->resInfo, pWrapper->topicHandle->schema.pSchema, pWrapper->topicHandle->schema.nCols); + + taosFreeQitem(pWrapper); return pRspObj; } -#if 0 -tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) { - tmq_message_t* msg = NULL; - for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { - SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); - for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { - SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); - int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT); - /*if (vgStatus != TMQ_VG_STATUS__IDLE) {*/ - /*continue;*/ - /*}*/ - SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg); - if (pReq == NULL) { - atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); - // TODO: out of mem - return NULL; - } - - SMqPollCbParam* pParam = taosMemoryMalloc(sizeof(SMqPollCbParam)); - if (pParam == NULL) { - atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); - // TODO: out of mem - return NULL; - } - pParam->tmq = tmq; - pParam->pVg = pVg; - pParam->epoch = tmq->epoch; - pParam->sync = 1; - pParam->msg = &msg; - tsem_init(&pParam->rspSem, 0, 0); - - SMsgSendInfo* sendInfo = taosMemoryMalloc(sizeof(SMsgSendInfo)); - if (sendInfo == NULL) { - return NULL; - } - - sendInfo->msgInfo = (SDataBuf){ - .pData = pReq, - .len = sizeof(SMqPollReq), - .handle = NULL, - }; - sendInfo->requestId = generateRequestId(); - sendInfo->requestObjRefId = 0; - sendInfo->param = pParam; - sendInfo->fp = tmqPollCb; - sendInfo->msgType = TDMT_VND_CONSUME; - - int64_t transporterId = 0; - /*printf("send poll\n");*/ - atomic_add_fetch_32(&tmq->waitingRequest, 1); - asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); - pVg->pollCnt++; - tmq->pollCnt++; - - tsem_wait(&pParam->rspSem); - tmq_message_t* nmsg = NULL; - while (1) { - taosReadQitem(tmq->mqueue, (void**)&nmsg); - if (nmsg == NULL) continue; - while (nmsg->head.mqMsgType != TMQ_MSG_TYPE__POLL_RSP) { - taosReadQitem(tmq->mqueue, (void**)&nmsg); - } - return nmsg; - } - } - } - return NULL; -} -#endif - int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { /*printf("call poll\n");*/ for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { @@ -1306,7 +1229,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { #endif } atomic_store_32(&pVg->vgSkipCnt, 0); - SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg); + SMqPollReqV2* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg); if (pReq == NULL) { atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); /*tsem_post(&tmq->rspSem);*/ @@ -1337,7 +1260,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { sendInfo->msgInfo = (SDataBuf){ .pData = pReq, - .len = sizeof(SMqPollReq), + .len = sizeof(SMqPollReqV2), .handle = NULL, }; sendInfo->requestId = pReq->reqId; @@ -1348,7 +1271,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { int64_t transporterId = 0; /*printf("send poll\n");*/ - atomic_add_fetch_32(&tmq->waitingRequest, 1); + /*atomic_add_fetch_32(&tmq->waitingRequest, 1);*/ tscDebug("consumer %ld send poll to %s : vg %d, epoch %d, req offset %ld, reqId %lu", tmq->consumerId, pTopic->topicName, pVg->vgId, tmq->epoch, pVg->currentOffset, pReq->reqId); /*printf("send vg %d %ld\n", pVg->vgId, pVg->currentOffset);*/ @@ -1390,7 +1313,7 @@ SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfReset) { if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RSP) { SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)rspWrapper; - atomic_sub_fetch_32(&tmq->readyRequest, 1); + /*atomic_sub_fetch_32(&tmq->readyRequest, 1);*/ /*printf("handle poll rsp %d\n", rspMsg->head.mqMsgType);*/ if (pollRspWrapper->msg.head.epoch == atomic_load_32(&tmq->epoch)) { /*printf("epoch match\n");*/ @@ -1398,7 +1321,7 @@ SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfReset) { /*printf("vg %d offset %ld up to %ld\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/ pVg->currentOffset = pollRspWrapper->msg.rspOffset; atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); - if (pollRspWrapper->msg.dataLen == 0) { + if (pollRspWrapper->msg.blockNum == 0) { taosFreeQitem(pollRspWrapper); rspWrapper = NULL; continue; @@ -1454,7 +1377,10 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { // TODO: put into another thread or delayed queue int64_t status = atomic_load_64(&tmq->status); - tmqAskEp(tmq, status == TMQ_CONSUMER_STATUS__INIT); + while (0 != tmqAskEp(tmq, status == TMQ_CONSUMER_STATUS__INIT)) { + tscDebug("not ready, retry\n"); + taosSsleep(1); + } rspObj = tmqHandleAllRsp(tmq, blocking_time, false); if (rspObj) { diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 38885333c9..4cf4a405e0 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -186,6 +186,7 @@ void mmInitMsgHandle(SMgmtWrapper *pWrapper) { dmSetMsgHandle(pWrapper, TDMT_MND_SUBSCRIBE, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_MQ_COMMIT_OFFSET, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_GET_SUB_EP, mmProcessReadMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_MQ_VG_CHANGE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_STREAM, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_GET_DB_CFG, mmProcessReadMsg, DEFAULT_HANDLE); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 751edd6f98..c9bc3514cc 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -321,7 +321,7 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) { dmSetMsgHandle(pWrapper, TDMT_VND_MQ_QUERY, (NodeMsgFp)vmProcessQueryMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CONNECT, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_MQ_DISCONNECT, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + /*dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);*/ dmSetMsgHandle(pWrapper, TDMT_VND_RES_READY, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); @@ -334,10 +334,11 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) { dmSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_CANCEL_SMA, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CANCEL_CONN, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + /*dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);*/ + /*dmSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);*/ + /*dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CANCEL_CONN, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);*/ + /*dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE);*/ + dmSetMsgHandle(pWrapper, TDMT_VND_MQ_VG_CHANGE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_CONSUME, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 8a66e5cb69..2ce7f0ae5f 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -127,7 +127,6 @@ static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) { pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer); if (pIter == NULL) break; - taosRLockLatch(&pConsumer->lock); int32_t hbStatus = atomic_add_fetch_32(&pConsumer->hbStatus, 1); int32_t status = atomic_load_32(&pConsumer->status); if (status == MQ_CONSUMER_STATUS__READY && hbStatus > MND_CONSUMER_LOST_HB_CNT) { @@ -143,6 +142,7 @@ static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) { if (status == MQ_CONSUMER_STATUS__LOST_REBD || status == MQ_CONSUMER_STATUS__READY) { // do nothing } else if (status == MQ_CONSUMER_STATUS__LOST) { + taosRLockLatch(&pConsumer->lock); int32_t topicNum = taosArrayGetSize(pConsumer->currentTopics); for (int32_t i = 0; i < topicNum; i++) { char key[TSDB_SUBSCRIBE_KEY_LEN]; @@ -151,7 +151,9 @@ static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) { SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId); } + taosRUnLockLatch(&pConsumer->lock); } else if (status == MQ_CONSUMER_STATUS__MODIFY) { + taosRLockLatch(&pConsumer->lock); int32_t newTopicNum = taosArrayGetSize(pConsumer->rebNewTopics); for (int32_t i = 0; i < newTopicNum; i++) { char key[TSDB_SUBSCRIBE_KEY_LEN]; @@ -169,11 +171,11 @@ static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) { SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId); } + taosRUnLockLatch(&pConsumer->lock); } else { // do nothing } - taosRUnLockLatch(&pConsumer->lock); mndReleaseConsumer(pMnode, pConsumer); } @@ -188,7 +190,7 @@ static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) { } else { taosHashCleanup(pRebMsg->rebSubHash); rpcFreeCont(pRebMsg); - mInfo("mq rebalance finished, no modification"); + mTrace("mq rebalance finished, no modification"); atomic_store_8(&mqInRebFlag, 0); } return 0; @@ -213,12 +215,12 @@ static int32_t mndProcessAskEpReq(SNodeMsg *pMsg) { // 1. check consumer status int32_t status = atomic_load_32(&pConsumer->status); + if (status == MQ_CONSUMER_STATUS__LOST) { - // recover consumer + // TODO: recover consumer } if (status != MQ_CONSUMER_STATUS__READY) { - mndReleaseConsumer(pMnode, pConsumer); terrno = TSDB_CODE_MND_CONSUMER_NOT_READY; return -1; } @@ -228,11 +230,13 @@ static int32_t mndProcessAskEpReq(SNodeMsg *pMsg) { // 2. check epoch, only send ep info when epoches do not match if (epoch != serverEpoch) { taosRLockLatch(&pConsumer->lock); + mInfo("process ask ep, consumer %ld(epoch %d), server epoch %d", consumerId, epoch, serverEpoch); int32_t numOfTopics = taosArrayGetSize(pConsumer->currentTopics); rsp.topics = taosArrayInit(numOfTopics, sizeof(SMqSubTopicEp)); if (rsp.topics == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; + taosRUnLockLatch(&pConsumer->lock); goto FAIL; } @@ -265,6 +269,7 @@ static int32_t mndProcessAskEpReq(SNodeMsg *pMsg) { topicEp.vgs = taosArrayInit(vgNum, sizeof(SMqSubVgEp)); if (topicEp.vgs == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; + taosRUnLockLatch(&pConsumer->lock); goto FAIL; } diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index dd2d6801e1..37c819ae60 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -250,13 +250,28 @@ void tDeleteSMqConsumerEpInSub(SMqConsumerEpInSub *pEpInSub) { int32_t tEncodeSMqConsumerEpInSub(void **buf, const SMqConsumerEpInSub *pEpInSub) { int32_t tlen = 0; tlen += taosEncodeFixedI64(buf, pEpInSub->consumerId); - tlen += taosEncodeArray(buf, pEpInSub->vgs, (FEncode)tEncodeSMqVgEp); + int32_t sz = taosArrayGetSize(pEpInSub->vgs); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + SMqVgEp *pVgEp = taosArrayGetP(pEpInSub->vgs, i); + tlen += tEncodeSMqVgEp(buf, pVgEp); + } + /*tlen += taosEncodeArray(buf, pEpInSub->vgs, (FEncode)tEncodeSMqVgEp);*/ return tlen; } void *tDecodeSMqConsumerEpInSub(const void *buf, SMqConsumerEpInSub *pEpInSub) { buf = taosDecodeFixedI64(buf, &pEpInSub->consumerId); - buf = taosDecodeArray(buf, &pEpInSub->vgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqSubVgEp)); + /*buf = taosDecodeArray(buf, &pEpInSub->vgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqSubVgEp));*/ + int32_t sz; + buf = taosDecodeFixedI32(buf, &sz); + pEpInSub->vgs = taosArrayInit(sz, sizeof(void *)); + for (int32_t i = 0; i < sz; i++) { + SMqVgEp *pVgEp = taosMemoryMalloc(sizeof(SMqVgEp)); + buf = tDecodeSMqVgEp(buf, pVgEp); + taosArrayPush(pEpInSub->vgs, &pVgEp); + } + return (void *)buf; } @@ -268,10 +283,12 @@ SMqSubscribeObj *tNewSubscribeObj(const char key[TSDB_SUBSCRIBE_KEY_LEN]) { pSubNew->vgNum = -1; pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); // TODO set free fp - SMqConsumerEpInSub *pEpInSub = taosMemoryMalloc(sizeof(SMqConsumerEpInSub)); - pEpInSub->vgs = taosArrayInit(0, sizeof(SMqVgEp)); + SMqConsumerEpInSub epInSub = { + .consumerId = -1, + .vgs = taosArrayInit(0, sizeof(void *)), + }; int64_t unexistKey = -1; - taosHashPut(pSubNew->consumerHash, &unexistKey, sizeof(int64_t), pEpInSub, sizeof(SMqConsumerEpInSub)); + taosHashPut(pSubNew->consumerHash, &unexistKey, sizeof(int64_t), &epInSub, sizeof(SMqConsumerEpInSub)); return pSubNew; } @@ -287,7 +304,7 @@ SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) { void *pIter = NULL; SMqConsumerEpInSub *pEpInSub = NULL; while (1) { - pIter = taosHashIterate(pSubNew->consumerHash, pIter); + pIter = taosHashIterate(pSub->consumerHash, pIter); if (pIter == NULL) break; pEpInSub = (SMqConsumerEpInSub *)pIter; SMqConsumerEpInSub newEp = { diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 5629421a84..b3583af1dc 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -434,7 +434,9 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib return -1; } - ASSERT(pSub->vgNum == 0); + ASSERT(pSub->vgNum == -1); + + pSub->vgNum = 0; int32_t levelNum = LIST_LENGTH(pPlan->pSubplans); if (levelNum != 1) { @@ -455,6 +457,9 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib int64_t unexistKey = -1; SMqConsumerEpInSub* pEpInSub = taosHashGet(pSub->consumerHash, &unexistKey, sizeof(int64_t)); + ASSERT(pEpInSub); + + ASSERT(taosHashGetSize(pSub->consumerHash) == 1); void* pIter = NULL; while (1) { @@ -492,10 +497,18 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib } taosArrayPush(pEpInSub->vgs, &pVgEp); + ASSERT(taosHashGetSize(pSub->consumerHash) == 1); + /*taosArrayPush(pSub->unassignedVg, &consumerEp);*/ } - taosHashRelease(pSub->consumerHash, pEpInSub); + ASSERT(pEpInSub->vgs->size > 0); + pEpInSub = taosHashGet(pSub->consumerHash, &unexistKey, sizeof(int64_t)); + + ASSERT(pEpInSub->vgs->size > 0); + + ASSERT(taosHashGetSize(pSub->consumerHash) == 1); + qDestroyQueryPlan(pPlan); return 0; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 48c9804619..7b2ec83230 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -109,6 +109,7 @@ static SMqSubscribeObj *mndCreateSub(SMnode *pMnode, const SMqTopicObj *pTopic, terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } + ASSERT(taosHashGetSize(pSub->consumerHash) == 1); if (mndSchedInitSubEp(pMnode, pTopic, pSub) < 0) { tDeleteSubscribeObj(pSub); @@ -116,6 +117,8 @@ static SMqSubscribeObj *mndCreateSub(SMnode *pMnode, const SMqTopicObj *pTopic, return NULL; } + ASSERT(taosHashGetSize(pSub->consumerHash) == 1); + return pSub; } @@ -255,7 +258,7 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const char *subK req.oldConsumerId = pRebVg->oldConsumerId; req.newConsumerId = pRebVg->newConsumerId; req.vgId = pRebVg->pVgEp->vgId; - req.qmsg = req.qmsg; + req.qmsg = pRebVg->pVgEp->qmsg; strncpy(req.subKey, subKey, TSDB_SUBSCRIBE_KEY_LEN); int32_t tlen = sizeof(SMsgHead) + tEncodeSMqRebVgReq(NULL, &req); @@ -663,14 +666,18 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR if (pInput->pTopic != NULL) { // create subscribe pOutput->pSub = mndCreateSub(pMnode, pInput->pTopic, pInput->pRebInfo->key); + ASSERT(taosHashGetSize(pOutput->pSub->consumerHash) == 1); } else { pOutput->pSub = tCloneSubscribeObj(pInput->pOldSub); } int32_t totalVgNum = pOutput->pSub->vgNum; + mInfo("mq rebalance subscription: %s, vgNum: %d", pOutput->pSub->key, pOutput->pSub->vgNum); + // 1. build temporary hash(vgId -> SMqRebOutputVg) to store modified vg SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + ASSERT(taosHashGetSize(pOutput->pSub->consumerHash) > 0); // 2. check and get actual removed consumers, put their vg into hash int32_t removedNum = taosArrayGetSize(pInput->pRebInfo->removedConsumers); int32_t actualRemoved = 0; @@ -692,18 +699,18 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); } - taosHashRelease(pOutput->pSub->consumerHash, pEpInSub); taosHashRemove(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t)); // put into removed taosArrayPush(pOutput->removedConsumers, &consumerId); } } ASSERT(removedNum == actualRemoved); + ASSERT(taosHashGetSize(pOutput->pSub->consumerHash) > 0); // if previously no consumer, there are vgs not assigned { - int64_t key = -1; - SMqConsumerEpInSub *pEpInSub = taosHashGet(pOutput->pSub->consumerHash, &key, sizeof(int64_t)); + int64_t unexistKey = -1; + SMqConsumerEpInSub *pEpInSub = taosHashGet(pOutput->pSub->consumerHash, &unexistKey, sizeof(int64_t)); ASSERT(pEpInSub); int32_t consumerVgNum = taosArrayGetSize(pEpInSub->vgs); for (int32_t i = 0; i < consumerVgNum; i++) { @@ -715,18 +722,22 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &rebOutput, sizeof(SMqRebOutputVg)); } - taosHashRelease(pOutput->pSub->consumerHash, pEpInSub); } // 3. calc vg number of each consumer - int32_t actualConsumerNum = taosHashGetSize(pInput->pOldSub->consumerHash) - 1 + - taosArrayGetSize(pInput->pRebInfo->newConsumers) - - taosArrayGetSize(pInput->pRebInfo->removedConsumers); - int32_t afterRebConsumerNum = taosHashGetSize(pOutput->pSub->consumerHash) - 1; - ASSERT(afterRebConsumerNum == actualConsumerNum); + int32_t oldSz = 0; + if (pInput->pOldSub) { + oldSz = taosHashGetSize(pInput->pOldSub->consumerHash) - 1; + } + int32_t afterRebConsumerNum = + oldSz + taosArrayGetSize(pInput->pRebInfo->newConsumers) - taosArrayGetSize(pInput->pRebInfo->removedConsumers); + int32_t minVgCnt = 0; + int32_t imbConsumerNum = 0; // calc num - int32_t minVgCnt = totalVgNum / actualConsumerNum; - int32_t imbConsumerNum = totalVgNum % actualConsumerNum; + if (afterRebConsumerNum) { + minVgCnt = totalVgNum / afterRebConsumerNum; + imbConsumerNum = totalVgNum % afterRebConsumerNum; + } // 4. first scan: remove consumer more than wanted, put to remove hash int32_t imbCnt = 0; @@ -735,6 +746,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); if (pIter == NULL) break; SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter; + if (pEpInSub->consumerId == -1) continue; ASSERT(pEpInSub->consumerId > 0); int32_t consumerVgNum = taosArrayGetSize(pEpInSub->vgs); // all old consumers still existing are touched @@ -783,6 +795,9 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR newConsumerEp.vgs = taosArrayInit(0, sizeof(void *)); taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, sizeof(SMqConsumerEpInSub)); + /*SMqConsumerEpInSub *pTestNew = taosHashGet(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));*/ + /*ASSERT(pTestNew->consumerId == consumerId);*/ + /*ASSERT(pTestNew->vgs == newConsumerEp.vgs);*/ taosArrayPush(pOutput->newConsumers, &consumerId); } } @@ -796,6 +811,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); if (pIter == NULL) break; SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter; + if (pEpInSub->consumerId == -1) continue; ASSERT(pEpInSub->consumerId > 0); /*int32_t consumerVgNum = taosArrayGetSize(pEpInSub->vgs);*/ if (imbCnt < imbConsumerNum) { @@ -833,8 +849,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR ASSERT(pRemovedIter == NULL); } else { // if all consumer is removed, put all vg into unassigned - int64_t key = -1; - SMqConsumerEpInSub *pEpInSub = taosHashGet(pOutput->pSub->consumerHash, &key, sizeof(int64_t)); + int64_t unexistKey = -1; + SMqConsumerEpInSub *pEpInSub = taosHashGet(pOutput->pSub->consumerHash, &unexistKey, sizeof(int64_t)); ASSERT(pEpInSub); ASSERT(pEpInSub->consumerId == -1); @@ -845,7 +861,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR if (pIter == NULL) break; pRebOutput = (SMqRebOutputVg *)pIter; ASSERT(pRebOutput->newConsumerId == -1); - taosArrayPush(pEpInSub->vgs, pRebOutput->pVgEp); + taosArrayPush(pEpInSub->vgs, &pRebOutput->pVgEp); + taosArrayPush(pOutput->rebVgs, pRebOutput); } } @@ -895,7 +912,7 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SNodeMsg *pMsg, const SMqRebO // 3.2 set new consumer consumerNum = taosArrayGetSize(pOutput->newConsumers); for (int32_t i = 0; i < consumerNum; i++) { - int64_t consumerId = *(int64_t *)taosArrayGet(pOutput->removedConsumers, i); + int64_t consumerId = *(int64_t *)taosArrayGet(pOutput->newConsumers, i); ASSERT(consumerId > 0); SMqConsumerObj *pConsumerOld = mndAcquireConsumer(pMnode, consumerId); SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumerOld->consumerId, pConsumerOld->cgroup); @@ -948,17 +965,22 @@ static int32_t mndProcessRebalanceReq(SNodeMsg *pMsg) { while (1) { pIter = taosHashIterate(pReq->rebSubHash, pIter); if (pIter == NULL) break; - SMqRebInputObj rebInput = {0}; - SMqRebOutputObj rebOutput = {0}; + SMqRebInputObj rebInput = {0}; + + SMqRebOutputObj rebOutput = {0}; + rebOutput.newConsumers = taosArrayInit(0, sizeof(void *)); + rebOutput.removedConsumers = taosArrayInit(0, sizeof(void *)); + rebOutput.touchedConsumers = taosArrayInit(0, sizeof(void *)); + rebOutput.rebVgs = taosArrayInit(0, sizeof(SMqRebOutputVg)); + SMqRebSubscribe *pRebSub = (SMqRebSubscribe *)pIter; SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pRebSub->key); if (pSub == NULL) { - taosRLockLatch(&pSub->lock); // split sub key and extract topic char topic[TSDB_TOPIC_FNAME_LEN]; char cgroup[TSDB_CGROUP_LEN]; - mndSplitSubscribeKey(pSub->key, topic, cgroup); + mndSplitSubscribeKey(pRebSub->key, topic, cgroup); SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); ASSERT(pTopic); taosRLockLatch(&pTopic->lock); @@ -968,24 +990,16 @@ static int32_t mndProcessRebalanceReq(SNodeMsg *pMsg) { rebInput.pRebInfo = pRebSub; rebInput.pOldSub = pSub; - int32_t unassignedVgNum = 0; - int64_t key = -1; - SMqConsumerEpInSub *pEpInSub = taosHashGet(pSub->consumerHash, &key, sizeof(int64_t)); - if (pEpInSub != NULL) { - ASSERT(pEpInSub->consumerId == key); - unassignedVgNum = taosArrayGetSize(pEpInSub->vgs); - } - mInfo("mq rebalance subscription: %s, vgNum: %d, unassignedVg: %d", pSub->key, pSub->vgNum, unassignedVgNum); - // TODO replace assert with error check ASSERT(mndDoRebalance(pMnode, &rebInput, &rebOutput) == 0); + ASSERT(taosArrayGetSize(rebOutput.rebVgs) != 0); ASSERT(mndPersistRebResult(pMnode, pMsg, &rebOutput) == 0); if (rebInput.pTopic) { SMqTopicObj *pTopic = (SMqTopicObj *)rebInput.pTopic; taosRUnLockLatch(&pTopic->lock); mndReleaseTopic(pMnode, pTopic); - taosRUnLockLatch(&pSub->lock); + } else { mndReleaseSubscribe(pMnode, pSub); } } diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 3ac1c522b2..5a61cefee1 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -22,12 +22,14 @@ #include "mndDb.h" #include "mndDnode.h" #include "mndFunc.h" +#include "mndGrant.h" #include "mndInfoSchema.h" -#include "mndPerfSchema.h" #include "mndMnode.h" #include "mndOffset.h" +#include "mndPerfSchema.h" #include "mndProfile.h" #include "mndQnode.h" +#include "mndQuery.h" #include "mndShow.h" #include "mndSma.h" #include "mndSnode.h" @@ -40,8 +42,6 @@ #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" -#include "mndQuery.h" -#include "mndGrant.h" #define MQ_TIMER_MS 3000 #define TRNAS_TIMER_MS 6000 diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index ed33473b16..5e074eeb12 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -16,6 +16,13 @@ #ifndef _TD_VNODE_TQ_H_ #define _TD_VNODE_TQ_H_ +#include "executor.h" +#include "os.h" +#include "thash.h" +#include "tmsg.h" +#include "ttimer.h" +#include "wal.h" + #ifdef __cplusplus extern "C" { #endif @@ -30,12 +37,6 @@ extern "C" { #define tqTrace(...) do { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0) // clang-format on -enum { - TQ_STREAM_TOKEN__DATA = 1, - TQ_STREAM_TOKEN__WATERMARK, - TQ_STREAM_TOKEN__CHECKPOINT, -}; - #define TQ_BUFFER_SIZE 4 #define TQ_BUCKET_MASK 0xFF @@ -151,22 +152,27 @@ typedef struct { } STqMetaStore; typedef struct { - SMemAllocatorFactory* pAllocatorFactory; - SMemAllocator* pAllocator; -} STqMemRef; + char subKey[TSDB_SUBSCRIBE_KEY_LEN]; + int64_t consumerId; + int32_t epoch; + char* qmsg; + // SRWLatch lock; + SWalReadHandle* pReadHandle; + // number should be identical to fetch thread num + qTaskInfo_t task[4]; +} STqExec; struct STQ { // the collection of groups // the handle of meta kvstore bool writeTrigger; char* path; - STqMemRef tqMemRef; STqMetaStore* tqMeta; - // STqPushMgr* tqPushMgr; - SHashObj* pStreamTasks; - SVnode* pVnode; - SWal* pWal; - SMeta* pVnodeMeta; + SHashObj* tqMetaNew; // subKey -> tqExec + SHashObj* pStreamTasks; + SVnode* pVnode; + SWal* pWal; + SMeta* pVnodeMeta; }; typedef struct { @@ -230,10 +236,6 @@ typedef struct { // TODO sync function } STqStreamPusher; -typedef struct { - int8_t type; // mq or stream -} STqPusher; - typedef struct { SHashObj* pHash; // } STqPushMgr; @@ -257,9 +259,10 @@ int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t version); int tqCommit(STQ*); int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId); -int32_t tqProcessSetConnReq(STQ* pTq, char* msg); -int32_t tqProcessRebReq(STQ* pTq, char* msg); -int32_t tqProcessCancelConnReq(STQ* pTq, char* msg); +int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen); +// int32_t tqProcessSetConnReq(STQ* pTq, char* msg); +// int32_t tqProcessRebReq(STQ* pTq, char* msg); +// int32_t tqProcessCancelConnReq(STQ* pTq, char* msg); int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId); int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t workerId); @@ -314,4 +317,4 @@ STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet } #endif -#endif /*_TD_VNODE_TQ_H_*/ \ No newline at end of file +#endif /*_TD_VNODE_TQ_H_*/ diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 0aa6023eaf..16c5a4752e 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -29,31 +29,14 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal, SMeta* pVnodeMeta, SMe pTq->pVnode = pVnode; pTq->pWal = pWal; pTq->pVnodeMeta = pVnodeMeta; -#if 0 - pTq->tqMemRef.pAllocatorFactory = allocFac; - pTq->tqMemRef.pAllocator = allocFac->create(allocFac); - if (pTq->tqMemRef.pAllocator == NULL) { - // TODO: error code of buffer pool - } -#endif pTq->tqMeta = tqStoreOpen(pTq, path, (FTqSerialize)tqSerializeConsumer, (FTqDeserialize)tqDeserializeConsumer, (FTqDelete)taosMemoryFree, 0); if (pTq->tqMeta == NULL) { taosMemoryFree(pTq); -#if 0 - allocFac->destroy(allocFac, pTq->tqMemRef.pAllocator); -#endif return NULL; } -#if 0 - pTq->tqPushMgr = tqPushMgrOpen(); - if (pTq->tqPushMgr == NULL) { - // free store - taosMemoryFree(pTq); - return NULL; - } -#endif + pTq->tqMetaNew = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK); pTq->pStreamTasks = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); @@ -248,16 +231,15 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu return 0; } -#if 0 + int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { - SMqPollReq* pReq = pMsg->pCont; - int64_t consumerId = pReq->consumerId; - int64_t fetchOffset; - int64_t blockingTime = pReq->blockingTime; - int32_t reqEpoch = pReq->epoch; + SMqPollReqV2* pReq = pMsg->pCont; + int64_t consumerId = pReq->consumerId; + int32_t reqEpoch = pReq->epoch; + int64_t fetchOffset; if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) { - fetchOffset = 0; + fetchOffset = walGetFirstVer(pTq->pWal); } else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__LATEST) { fetchOffset = walGetLastVer(pTq->pWal); } else { @@ -267,65 +249,29 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { vDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch, TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset); - SMqPollRsp rsp = { - /*.consumerId = consumerId,*/ - .numOfTopics = 0, - .pBlockData = NULL, - }; + STqExec* pExec = taosHashGet(pTq->tqMetaNew, pReq->subKey, strlen(pReq->subKey)); + ASSERT(pExec); - STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, consumerId); - if (pConsumer == NULL) { - vWarn("tmq poll: consumer %ld (epoch %d) not found in vg %d", consumerId, pReq->epoch, TD_VID(pTq->pVnode)); - pMsg->pCont = NULL; - pMsg->contLen = 0; - pMsg->code = -1; - tmsgSendRsp(pMsg); - return 0; - } - - int32_t consumerEpoch = atomic_load_32(&pConsumer->epoch); + int32_t consumerEpoch = atomic_load_32(&pExec->epoch); while (consumerEpoch < reqEpoch) { - consumerEpoch = atomic_val_compare_exchange_32(&pConsumer->epoch, consumerEpoch, reqEpoch); + consumerEpoch = atomic_val_compare_exchange_32(&pExec->epoch, consumerEpoch, reqEpoch); } - STqTopic* pTopic = NULL; - int32_t sz = taosArrayGetSize(pConsumer->topics); - for (int32_t i = 0; i < sz; i++) { - STqTopic* topic = taosArrayGet(pConsumer->topics, i); - // TODO race condition - ASSERT(pConsumer->consumerId == consumerId); - if (strcmp(topic->topicName, pReq->topic) == 0) { - pTopic = topic; - break; - } - } - if (pTopic == NULL) { - vWarn("tmq poll: consumer %ld (epoch %d) topic %s not found in vg %d", consumerId, pReq->epoch, pReq->topic, - TD_VID(pTq->pVnode)); - pMsg->pCont = NULL; - pMsg->contLen = 0; - pMsg->code = -1; - tmsgSendRsp(pMsg); - return 0; - } - - vDebug("poll topic %s from consumer %ld (epoch %d) vg %d", pTopic->topicName, consumerId, pReq->epoch, - TD_VID(pTq->pVnode)); - + SMqDataBlkRsp rsp = {0}; rsp.reqOffset = pReq->currentOffset; - rsp.skipLogNum = 0; + rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t)); + rsp.blockData = taosArrayInit(0, sizeof(void*)); while (1) { - /*if (fetchOffset > walGetLastVer(pTq->pWal) || walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) {*/ - // TODO - consumerEpoch = atomic_load_32(&pConsumer->epoch); + consumerEpoch = atomic_load_32(&pExec->epoch); if (consumerEpoch > reqEpoch) { vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d discard req epoch %d", consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, consumerEpoch, reqEpoch); break; } + SWalReadHead* pHead; - if (walReadWithHandle_s(pTopic->pReadhandle, fetchOffset, &pHead) < 0) { + if (walReadWithHandle_s(pExec->pReadHandle, fetchOffset, &pHead) < 0) { // TODO: no more log, set timer to wait blocking time // if data inserted during waiting, launch query and // response to user @@ -333,101 +279,86 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { TD_VID(pTq->pVnode), fetchOffset); break; } + vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, pHead->msgType); - /*int8_t pos = fetchOffset % TQ_BUFFER_SIZE;*/ - /*pHead = pTopic->pReadhandle->pHead;*/ + if (pHead->msgType == TDMT_VND_SUBMIT) { SSubmitReq* pCont = (SSubmitReq*)&pHead->body; - qTaskInfo_t task = pTopic->buffer.output[workerId].task; + qTaskInfo_t task = pExec->task[workerId]; ASSERT(task); qSetStreamInput(task, pCont, STREAM_DATA_TYPE_SUBMIT_BLOCK); - SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); while (1) { SSDataBlock* pDataBlock = NULL; - uint64_t ts; + uint64_t ts = 0; if (qExecTask(task, &pDataBlock, &ts) < 0) { - ASSERT(false); - } - if (pDataBlock == NULL) { - /*pos = fetchOffset % TQ_BUFFER_SIZE;*/ - break; + ASSERT(0); } + if (pDataBlock == NULL) break; - taosArrayPush(pRes, pDataBlock); + ASSERT(pDataBlock->info.rows != 0); + ASSERT(pDataBlock->info.numOfCols != 0); + + int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pDataBlock); + void* buf = taosMemoryCalloc(1, dataStrLen); + SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)buf; + pRetrieve->useconds = ts; + pRetrieve->precision = TSDB_DEFAULT_PRECISION; + pRetrieve->compressed = 0; + pRetrieve->completed = 1; + pRetrieve->numOfRows = htonl(pDataBlock->info.rows); + + // TODO enable compress + int32_t actualLen = 0; + blockCompressEncode(pDataBlock, pRetrieve->data, &actualLen, pDataBlock->info.numOfCols, false); + actualLen += sizeof(SRetrieveTableRsp); + ASSERT(actualLen <= dataStrLen); + taosArrayPush(rsp.blockDataLen, &actualLen); + taosArrayPush(rsp.blockData, &buf); + rsp.blockNum++; } - - if (taosArrayGetSize(pRes) == 0) { - vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d skip log %ld since not wanted", consumerId, - pReq->epoch, TD_VID(pTq->pVnode), fetchOffset); - fetchOffset++; - rsp.skipLogNum++; - taosArrayDestroy(pRes); - continue; - } - rsp.schema = pTopic->buffer.output[workerId].pReadHandle->pSchemaWrapper; - rsp.rspOffset = fetchOffset; - - rsp.numOfTopics = 1; - rsp.pBlockData = pRes; - - int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqPollRsp(NULL, &rsp); - void* buf = rpcMallocCont(tlen); - if (buf == NULL) { - pMsg->code = -1; - taosMemoryFree(pHead); - return -1; - } - ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP; - ((SMqRspHead*)buf)->epoch = pReq->epoch; - ((SMqRspHead*)buf)->consumerId = consumerId; - - void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); - tEncodeSMqPollRsp(&abuf, &rsp); - /*taosArrayDestroyEx(rsp.pBlockData, (void (*)(void*))tDeleteSSDataBlock);*/ - pMsg->pCont = buf; - pMsg->contLen = tlen; - pMsg->code = 0; - vDebug("vg %d offset %ld msgType %d from consumer %ld (epoch %d) actual rsp", TD_VID(pTq->pVnode), fetchOffset, - pHead->msgType, consumerId, pReq->epoch); - tmsgSendRsp(pMsg); - taosMemoryFree(pHead); - return 0; - } else { - taosMemoryFree(pHead); - fetchOffset++; - rsp.skipLogNum++; } + + // TODO batch optimization + if (rsp.blockNum != 0) break; + rsp.skipLogNum++; + fetchOffset++; } - /*if (blockingTime != 0) {*/ - /*tqAddClientPusher(pTq->tqPushMgr, pMsg, consumerId, blockingTime);*/ - /*} else {*/ - int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqPollRsp(NULL, &rsp); + ASSERT(taosArrayGetSize(rsp.blockData) == rsp.blockNum); + ASSERT(taosArrayGetSize(rsp.blockDataLen) == rsp.blockNum); + + if (rsp.blockNum != 0) + rsp.rspOffset = fetchOffset; + else + rsp.rspOffset = fetchOffset - 1; + + int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqDataBlkRsp(NULL, &rsp); void* buf = rpcMallocCont(tlen); if (buf == NULL) { pMsg->code = -1; return -1; } + ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP; ((SMqRspHead*)buf)->epoch = pReq->epoch; - rsp.rspOffset = fetchOffset - 1; + ((SMqRspHead*)buf)->consumerId = consumerId; void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); - tEncodeSMqPollRsp(&abuf, &rsp); - rsp.pBlockData = NULL; + tEncodeSMqDataBlkRsp(&abuf, &rsp); pMsg->pCont = buf; pMsg->contLen = tlen; pMsg->code = 0; tmsgSendRsp(pMsg); - vDebug("vg %d offset %ld from consumer %ld (epoch %d) not rsp", TD_VID(pTq->pVnode), fetchOffset, consumerId, - pReq->epoch); - /*}*/ + vDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld", + TD_VID(pTq->pVnode), fetchOffset, consumerId, pReq->epoch, rsp.blockNum, rsp.reqOffset, rsp.rspOffset); + + // TODO destroy return 0; } -#endif +#if 0 int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { SMqPollReq* pReq = pMsg->pCont; int64_t consumerId = pReq->consumerId; @@ -436,7 +367,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { int32_t reqEpoch = pReq->epoch; if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) { - fetchOffset = 0; + fetchOffset = walGetFirstVer(pTq->pWal); } else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__LATEST) { fetchOffset = walGetLastVer(pTq->pWal); } else { @@ -635,7 +566,56 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { return 0; } +#endif +// TODO: persist meta into tdb +int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { + SMqRebVgReq req; + tDecodeSMqRebVgReq(msg, &req); + // todo lock + STqExec* pExec = taosHashGet(pTq->tqMetaNew, req.subKey, strlen(req.subKey)); + if (pExec == NULL) { + ASSERT(req.oldConsumerId == -1); + ASSERT(req.newConsumerId != -1); + STqExec exec = {0}; + pExec = &exec; + /*taosInitRWLatch(&pExec->lock);*/ + + memcpy(pExec->subKey, req.subKey, TSDB_SUBSCRIBE_KEY_LEN); + pExec->consumerId = req.newConsumerId; + pExec->epoch = -1; + pExec->qmsg = req.qmsg; + req.qmsg = NULL; + pExec->pReadHandle = walOpenReadHandle(pTq->pVnode->pWal); + for (int32_t i = 0; i < 4; i++) { + STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnodeMeta); + SReadHandle handle = { + .reader = pReadHandle, + .meta = pTq->pVnodeMeta, + }; + pExec->task[i] = qCreateStreamExecTaskInfo(pExec->qmsg, &handle); + ASSERT(pExec->task[i]); + } + taosHashPut(pTq->tqMetaNew, req.subKey, strlen(req.subKey), pExec, sizeof(STqExec)); + return 0; + } else { + if (req.newConsumerId != -1) { + /*taosWLockLatch(&pExec->lock);*/ + ASSERT(pExec->consumerId == req.oldConsumerId); + // TODO handle qmsg and exec modification + atomic_store_64(&pExec->consumerId, req.newConsumerId); + atomic_add_fetch_32(&pExec->epoch, 1); + /*taosWUnLockLatch(&pExec->lock);*/ + return 0; + } else { + // TODO + /*taosHashRemove(pTq->tqMetaNew, req.subKey, strlen(req.subKey));*/ + return 0; + } + } +} + +#if 0 int32_t tqProcessRebReq(STQ* pTq, char* msg) { SMqMVRebReq req = {0}; terrno = TSDB_CODE_SUCCESS; @@ -754,6 +734,7 @@ int32_t tqProcessCancelConnReq(STQ* pTq, char* msg) { terrno = TSDB_CODE_SUCCESS; return 0; } +#endif int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int32_t parallel) { if (pTask->execType == TASK_EXEC__NONE) return 0; diff --git a/source/dnode/vnode/src/tq/tqMetaStore.c b/source/dnode/vnode/src/tq/tqMetaStore.c index 357917e0ba..809fd7cb06 100644 --- a/source/dnode/vnode/src/tq/tqMetaStore.c +++ b/source/dnode/vnode/src/tq/tqMetaStore.c @@ -13,7 +13,6 @@ * along with this program. If not, see . */ #include "vnodeInt.h" -// TODO:replace by an abstract file layer // #include // #include // #include diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index fface9d6c5..1486d18b5c 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -80,6 +80,13 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg pRsp->msgType = TDMT_VND_SUBMIT_RSP; vnodeProcessSubmitReq(pVnode, ptr, pRsp); break; + case TDMT_VND_MQ_VG_CHANGE: + if (tqProcessVgChangeReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), + pMsg->contLen - sizeof(SMsgHead)) < 0) { + // TODO: handle error + } + break; +#if 0 case TDMT_VND_MQ_SET_CONN: { if (tqProcessSetConnReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) { // TODO: handle error @@ -93,6 +100,7 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg if (tqProcessCancelConnReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) { } } break; +#endif case TDMT_VND_TASK_DEPLOY: { if (tqProcessTaskDeploy(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), pMsg->contLen - sizeof(SMsgHead)) < 0) { @@ -307,4 +315,4 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pSubmitReq, SRpcMsg pRsp->contLen = sizeof(SSubmitRsp); return 0; -} \ No newline at end of file +} diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index c2c4af18e5..3d3d13fcb6 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -38,7 +38,7 @@ extern int openU(const char *, int, ...); /* MsvcLibX UTF-8 version of open */ #include #if !defined(_TD_DARWIN_64) - #include +#include #endif #include #include @@ -58,9 +58,9 @@ typedef int32_t FileFd; typedef struct TdFile { TdThreadRwlock rwlock; - int refId; - FileFd fd; - FILE *fp; + int refId; + FileFd fd; + FILE *fp; } * TdFilePtr, TdFile; #define FILE_WITH_LOCK 1 @@ -182,7 +182,7 @@ int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) { return 0; #else struct stat fileStat; - int32_t code = stat(path, &fileStat); + int32_t code = stat(path, &fileStat); if (code < 0) { return code; } @@ -203,7 +203,7 @@ int32_t taosDevInoFile(const char *path, int64_t *stDev, int64_t *stIno) { return 0; #else struct stat fileStat; - int32_t code = stat(path, &fileStat); + int32_t code = stat(path, &fileStat); if (code < 0) { return code; } @@ -226,7 +226,7 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) return NULL; #else - int fd = -1; + int fd = -1; FILE *fp = NULL; if (tdFileOptions & TD_FILE_STREAM) { char *mode = NULL; @@ -325,7 +325,7 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { #if FILE_WITH_LOCK taosThreadRwlockRdlock(&(pFile->rwlock)); #endif - assert(pFile->fd >= 0); // Please check if you have closed the file. + assert(pFile->fd >= 0); // Please check if you have closed the file. int64_t leftbytes = count; int64_t readbytes; char *tbuf = (char *)buf; @@ -365,7 +365,7 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) #if FILE_WITH_LOCK taosThreadRwlockRdlock(&(pFile->rwlock)); #endif - assert(pFile->fd >= 0); // Please check if you have closed the file. + assert(pFile->fd >= 0); // Please check if you have closed the file. int64_t ret = pread(pFile->fd, buf, count, offset); #if FILE_WITH_LOCK taosThreadRwlockUnlock(&(pFile->rwlock)); @@ -380,7 +380,7 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { #if FILE_WITH_LOCK taosThreadRwlockWrlock(&(pFile->rwlock)); #endif - assert(pFile->fd >= 0); // Please check if you have closed the file. + /*assert(pFile->fd >= 0); // Please check if you have closed the file.*/ int64_t nleft = count; int64_t nwritten = 0; @@ -414,7 +414,7 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { #if FILE_WITH_LOCK taosThreadRwlockRdlock(&(pFile->rwlock)); #endif - assert(pFile->fd >= 0); // Please check if you have closed the file. + assert(pFile->fd >= 0); // Please check if you have closed the file. int64_t ret = lseek(pFile->fd, (long)offset, whence); #if FILE_WITH_LOCK taosThreadRwlockUnlock(&(pFile->rwlock)); @@ -429,10 +429,10 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { if (pFile == NULL) { return 0; } - assert(pFile->fd >= 0); // Please check if you have closed the file. + assert(pFile->fd >= 0); // Please check if you have closed the file. struct stat fileStat; - int32_t code = fstat(pFile->fd, &fileStat); + int32_t code = fstat(pFile->fd, &fileStat); if (code < 0) { return code; } @@ -456,7 +456,7 @@ int32_t taosLockFile(TdFilePtr pFile) { if (pFile == NULL) { return 0; } - assert(pFile->fd >= 0); // Please check if you have closed the file. + assert(pFile->fd >= 0); // Please check if you have closed the file. return (int32_t)flock(pFile->fd, LOCK_EX | LOCK_NB); #endif @@ -469,7 +469,7 @@ int32_t taosUnLockFile(TdFilePtr pFile) { if (pFile == NULL) { return 0; } - assert(pFile->fd >= 0); // Please check if you have closed the file. + assert(pFile->fd >= 0); // Please check if you have closed the file. return (int32_t)flock(pFile->fd, LOCK_UN | LOCK_NB); #endif @@ -529,7 +529,7 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { if (pFile == NULL) { return 0; } - assert(pFile->fd >= 0); // Please check if you have closed the file. + assert(pFile->fd >= 0); // Please check if you have closed the file. return ftruncate(pFile->fd, l_size); #endif @@ -637,7 +637,7 @@ int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t co } off_t len = count; while (len > 0) { - char buf[1024 * 16]; + char buf[1024 * 16]; off_t n = sizeof(buf); if (len < n) n = len; size_t m = fread(buf, 1, n, in_file); @@ -662,7 +662,7 @@ int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t count) { } off_t len = count; while (len > 0) { - char buf[1024 * 16]; + char buf[1024 * 16]; off_t n = sizeof(buf); if (len < n) n = len; size_t m = read(sfd, buf, n); @@ -750,7 +750,7 @@ void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length) { if (pFile == NULL) { return NULL; } - assert(pFile->fd >= 0); // Please check if you have closed the file. + assert(pFile->fd >= 0); // Please check if you have closed the file. void *ptr = mmap(NULL, length, PROT_READ, MAP_SHARED, pFile->fd, 0); return ptr; @@ -811,4 +811,4 @@ bool taosCheckAccessFile(const char *pathname, int32_t tdFileAccessOptions) { bool taosCheckExistFile(const char *pathname) { return taosCheckAccessFile(pathname, TD_FILE_ACCESS_EXIST_OK); }; -#endif // WINDOWS +#endif // WINDOWS diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 0181b63380..1c655fc2bf 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -76,7 +76,7 @@ int32_t taosArrayEnsureCap(SArray* pArray, size_t newCap) { } void* taosArrayAddBatch(SArray* pArray, const void* pData, int32_t nEles) { - if (pArray == NULL || pData == NULL) { + if (pData == NULL) { return NULL; } @@ -318,7 +318,6 @@ void taosArrayClearEx(SArray* pArray, void (*fp)(void*)) { pArray->size = 0; } - void* taosArrayDestroy(SArray* pArray) { if (pArray) { taosMemoryFree(pArray->pData); @@ -445,7 +444,8 @@ static void taosArrayInsertSort(SArray* pArray, __ext_compar_fn_t fn, const void } SArray* taosArrayDeepCopy(const SArray* pSrc, FCopy deepCopy) { - SArray* pArray = taosArrayInit(pSrc->size, pSrc->elemSize); + ASSERT(pSrc->elemSize == sizeof(void*)); + SArray* pArray = taosArrayInit(pSrc->size, sizeof(void*)); for (int32_t i = 0; i < pSrc->size; i++) { void* clone = deepCopy(taosArrayGetP(pSrc, i)); taosArrayPush(pArray, &clone); @@ -471,6 +471,7 @@ void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t for (int32_t i = 0; i < sz; i++) { void* data = taosMemoryCalloc(1, dataSz); buf = decode(buf, data); + taosArrayPush(*pArray, &data); } return (void*)buf; } diff --git a/tests/script/tsim/tmq/basic1.sim b/tests/script/tsim/tmq/basic1.sim index f7bd273624..60064b174e 100644 --- a/tests/script/tsim/tmq/basic1.sim +++ b/tests/script/tsim/tmq/basic1.sim @@ -33,8 +33,8 @@ endi sql connect $dbNamme = d0 -print =============== create database , vgroup 1 -sql create database $dbNamme vgroups 1 +print =============== create database , vgroup 4 +sql create database $dbNamme vgroups 4 sql use $dbNamme print =============== create super table diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index 5ea15a6622..9b627e187c 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -276,7 +276,7 @@ tmq_t* build_consumer() { } tmq_conf_set(conf, "td.connect.user", "root"); tmq_conf_set(conf, "td.connect.pass", "taosdata"); - tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName); + ASSERT(TMQ_CONF_OK == tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName)); tmq_t* tmq = tmq_consumer_new1(conf, NULL, 0); assert(tmq); tmq_conf_destroy(conf); @@ -437,12 +437,14 @@ void* threadFunc(void* param) { pInfo->consumeMsgCnt = parallel_consume(tmq, 1); //} +#if 0 err = tmq_unsubscribe(tmq); if (err) { printf("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err)); pInfo->consumeMsgCnt = -1; return NULL; } +#endif return NULL; } @@ -485,11 +487,13 @@ int main(int32_t argc, char* argv[]) { totalMsgs = parallel_consume(tmq, 0); } +#if 0 err = tmq_unsubscribe(tmq); if (err) { printf("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err)); exit(-1); } +#endif if (g_stConfInfo.numOfTopic1) { for (int32_t i = 0; i < numOfThreads; i++) { From f7a63661ac5f592093b4ab3b45aa0e7a14db8a80 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 20 Apr 2022 22:08:04 +0800 Subject: [PATCH 43/47] fix --- source/dnode/mnode/impl/src/mndConsumer.c | 8 ++++---- tests/test/c/tmqSim.c | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 2ce7f0ae5f..7f5df5d356 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -376,7 +376,7 @@ static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) { if (mndTransPrepare(pMnode, pTrans) != 0) goto SUBSCRIBE_OVER; } else { - taosRLockLatch(&pConsumerOld->lock); + /*taosRLockLatch(&pConsumerOld->lock);*/ int32_t status = atomic_load_32(&pConsumerOld->status); if (status != MQ_CONSUMER_STATUS__READY) { terrno = TSDB_CODE_MND_CONSUMER_NOT_READY; @@ -440,7 +440,7 @@ static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) { SUBSCRIBE_OVER: if (pConsumerOld) { - taosRUnLockLatch(&pConsumerOld->lock); + /*taosRUnLockLatch(&pConsumerOld->lock);*/ mndReleaseConsumer(pMnode, pConsumerOld); } if (pConsumerNew) { @@ -628,9 +628,9 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, // remove from removed topic for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->rebRemovedTopics); i++) { - char *topic = taosArrayGetP(pOldConsumer->rebNewTopics, i); + char *topic = taosArrayGetP(pOldConsumer->rebRemovedTopics, i); if (strcmp(removedTopic, topic) == 0) { - taosArrayRemove(pOldConsumer->rebNewTopics, i); + taosArrayRemove(pOldConsumer->rebRemovedTopics, i); taosMemoryFree(topic); break; } diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index 9b627e187c..1b8c0d527d 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -437,7 +437,7 @@ void* threadFunc(void* param) { pInfo->consumeMsgCnt = parallel_consume(tmq, 1); //} -#if 0 +#if 1 err = tmq_unsubscribe(tmq); if (err) { printf("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err)); @@ -487,7 +487,7 @@ int main(int32_t argc, char* argv[]) { totalMsgs = parallel_consume(tmq, 0); } -#if 0 +#if 1 err = tmq_unsubscribe(tmq); if (err) { printf("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err)); From ce6459a9f859b7ceb19ae4e9aef13adf3f1fcb7f Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 21 Apr 2022 01:47:29 +0800 Subject: [PATCH 44/47] fix(tmq): rebalance --- source/dnode/mnode/impl/inc/mndOffset.h | 2 +- source/dnode/mnode/impl/src/mndOffset.c | 18 ++++++++--- source/dnode/mnode/impl/src/mndSubscribe.c | 36 +++++++++++++++++++--- source/dnode/vnode/src/tq/tq.c | 29 +++++++++-------- 4 files changed, 62 insertions(+), 23 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndOffset.h b/source/dnode/mnode/impl/inc/mndOffset.h index e18eec973f..556bfe6a53 100644 --- a/source/dnode/mnode/impl/inc/mndOffset.h +++ b/source/dnode/mnode/impl/inc/mndOffset.h @@ -31,7 +31,7 @@ void mndReleaseOffset(SMnode *pMnode, SMqOffsetObj *pOffset); SSdbRaw *mndOffsetActionEncode(SMqOffsetObj *pOffset); SSdbRow *mndOffsetActionDecode(SSdbRaw *pRaw); -int32_t mndCreateOffset(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs); +int32_t mndCreateOffsets(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs); static FORCE_INLINE int32_t mndMakePartitionKey(char *key, const char *cgroup, const char *topicName, int32_t vgId) { return snprintf(key, TSDB_PARTITION_KEY_LEN, "%d:%s:%s", vgId, cgroup, topicName); diff --git a/source/dnode/mnode/impl/src/mndOffset.c b/source/dnode/mnode/impl/src/mndOffset.c index 31c16e1e1d..dad912b4e6 100644 --- a/source/dnode/mnode/impl/src/mndOffset.c +++ b/source/dnode/mnode/impl/src/mndOffset.c @@ -130,8 +130,7 @@ OFFSET_DECODE_OVER: return pRow; } -int32_t mndCreateOffset(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs) { - int32_t code = 0; +int32_t mndCreateOffsets(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs) { int32_t sz = taosArrayGetSize(vgs); for (int32_t i = 0; i < sz; i++) { SMqConsumerEp *pConsumerEp = taosArrayGet(vgs, i); @@ -170,13 +169,22 @@ static int32_t mndProcessCommitOffsetReq(SNodeMsg *pMsg) { if (mndMakePartitionKey(key, pOffset->cgroup, pOffset->topicName, pOffset->vgId) < 0) { return -1; } + bool create = false; SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, key); - ASSERT(pOffsetObj); + if (pOffsetObj == NULL) { + pOffsetObj = taosMemoryMalloc(sizeof(SMqOffset)); + memcpy(pOffsetObj->key, key, TSDB_PARTITION_KEY_LEN); + create = true; + } pOffsetObj->offset = pOffset->offset; SSdbRaw *pOffsetRaw = mndOffsetActionEncode(pOffsetObj); sdbSetRawStatus(pOffsetRaw, SDB_STATUS_READY); mndTransAppendRedolog(pTrans, pOffsetRaw); - mndReleaseOffset(pMnode, pOffsetObj); + if (create) { + taosMemoryFree(pOffsetObj); + } else { + mndReleaseOffset(pMnode, pOffsetObj); + } } if (mndTransPrepare(pMnode, pTrans) != 0) { @@ -201,7 +209,7 @@ static int32_t mndOffsetActionDelete(SSdb *pSdb, SMqOffsetObj *pOffset) { static int32_t mndOffsetActionUpdate(SSdb *pSdb, SMqOffsetObj *pOldOffset, SMqOffsetObj *pNewOffset) { mTrace("offset:%s, perform update action", pOldOffset->key); - pOldOffset->offset = pNewOffset->offset; + atomic_store_64(&pOldOffset->offset, pNewOffset->offset); return 0; } diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 7b2ec83230..f708d4ffc1 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -813,6 +813,20 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter; if (pEpInSub->consumerId == -1) continue; ASSERT(pEpInSub->consumerId > 0); + + // push until equal minVg + while (taosArrayGetSize(pEpInSub->vgs) < minVgCnt) { + // iter hash and find one vg + pRemovedIter = taosHashIterate(pHash, pRemovedIter); + ASSERT(pRemovedIter); + pRebVg = (SMqRebOutputVg *)pRemovedIter; + // push + taosArrayPush(pEpInSub->vgs, &pRebVg->pVgEp); + pRebVg->newConsumerId = pEpInSub->consumerId; + taosArrayPush(pOutput->rebVgs, pRebVg); + } + +#if 0 /*int32_t consumerVgNum = taosArrayGetSize(pEpInSub->vgs);*/ if (imbCnt < imbConsumerNum) { imbCnt++; @@ -840,13 +854,25 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR taosArrayPush(pOutput->rebVgs, pRebVg); } } +#endif } // 7. handle unassigned vg if (taosHashGetSize(pOutput->pSub->consumerHash) != 1) { - // if has consumer, vg should be all assigned - pRemovedIter = taosHashIterate(pHash, pRemovedIter); - ASSERT(pRemovedIter == NULL); + // if has consumer, assign all left vg + while (1) { + pRemovedIter = taosHashIterate(pHash, pRemovedIter); + if (pRemovedIter == NULL) break; + pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); + ASSERT(pIter); + pRebVg = (SMqRebOutputVg *)pRemovedIter; + SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter; + if (pEpInSub->consumerId == -1) continue; + ASSERT(pEpInSub->consumerId > 0); + taosArrayPush(pEpInSub->vgs, &pRebVg->pVgEp); + pRebVg->newConsumerId = pEpInSub->consumerId; + taosArrayPush(pOutput->rebVgs, pRebVg); + } } else { // if all consumer is removed, put all vg into unassigned int64_t unexistKey = -1; @@ -992,7 +1018,9 @@ static int32_t mndProcessRebalanceReq(SNodeMsg *pMsg) { // TODO replace assert with error check ASSERT(mndDoRebalance(pMnode, &rebInput, &rebOutput) == 0); - ASSERT(taosArrayGetSize(rebOutput.rebVgs) != 0); + // if add more consumer to balanced subscribe, + // possibly no vg is changed + /*ASSERT(taosArrayGetSize(rebOutput.rebVgs) != 0);*/ ASSERT(mndPersistRebResult(pMnode, pMsg, &rebOutput) == 0); if (rebInput.pTopic) { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 16c5a4752e..c7c8054120 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -355,6 +355,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { TD_VID(pTq->pVnode), fetchOffset, consumerId, pReq->epoch, rsp.blockNum, rsp.reqOffset, rsp.rspOffset); // TODO destroy + taosArrayDestroy(rsp.blockData); + taosArrayDestroy(rsp.blockDataLen); return 0; } @@ -599,19 +601,20 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { taosHashPut(pTq->tqMetaNew, req.subKey, strlen(req.subKey), pExec, sizeof(STqExec)); return 0; } else { - if (req.newConsumerId != -1) { - /*taosWLockLatch(&pExec->lock);*/ - ASSERT(pExec->consumerId == req.oldConsumerId); - // TODO handle qmsg and exec modification - atomic_store_64(&pExec->consumerId, req.newConsumerId); - atomic_add_fetch_32(&pExec->epoch, 1); - /*taosWUnLockLatch(&pExec->lock);*/ - return 0; - } else { - // TODO - /*taosHashRemove(pTq->tqMetaNew, req.subKey, strlen(req.subKey));*/ - return 0; - } + /*if (req.newConsumerId != -1) {*/ + /*taosWLockLatch(&pExec->lock);*/ + ASSERT(pExec->consumerId == req.oldConsumerId); + // TODO handle qmsg and exec modification + atomic_store_32(&pExec->epoch, -1); + atomic_store_64(&pExec->consumerId, req.newConsumerId); + atomic_add_fetch_32(&pExec->epoch, 1); + /*taosWUnLockLatch(&pExec->lock);*/ + return 0; + /*} else {*/ + // TODO + /*taosHashRemove(pTq->tqMetaNew, req.subKey, strlen(req.subKey));*/ + /*return 0;*/ + /*}*/ } } From c4f02d6dfafc1c8b8760033c358ab3065a981d29 Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Thu, 21 Apr 2022 09:54:50 +0800 Subject: [PATCH 45/47] put python test cases in Jenkins --- Jenkinsfile2 | 11 +++++++++++ tests/system-test/fulltest.sh | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 441cf89626..c9a8c36363 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -88,6 +88,12 @@ def pre_test(){ cmake .. > /dev/null make -j4> /dev/null ''' + sh''' + cd ${WKPY} + git reset --hard + git pull + pip3 install . + ''' return 1 } @@ -97,6 +103,7 @@ pipeline { environment{ WK = '/var/lib/jenkins/workspace/TDinternal' WKC= '/var/lib/jenkins/workspace/TDengine' + WKPY= '/var/lib/jenkins/workspace/taos-connector-python' } stages { stage('pre_build'){ @@ -117,6 +124,10 @@ pipeline { ./test-all.sh b1fq ''' sh''' + cd ${WKC}/tests/system-test + ./fulltest.sh + ''' + sh''' cd ${WKC}/debug ctest ''' diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 0b8bbd24b2..3a462051bc 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -1,4 +1,4 @@ -python3 ./test.py -f 2-query/between.py +python3 ./test.py -f 2-query/varchar.py From 4b982805f8057693af24ba8e7bc405098c199067 Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Thu, 21 Apr 2022 10:15:28 +0800 Subject: [PATCH 46/47] set LD_LIBRARY_PATH for python test --- Jenkinsfile2 | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index c9a8c36363..a2043797c0 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -124,6 +124,7 @@ pipeline { ./test-all.sh b1fq ''' sh''' + export LD_LIBRARY_PATH=${WKC}/debug/build/lib cd ${WKC}/tests/system-test ./fulltest.sh ''' From 3ae976988733b42b13df7ef0594c424d914a047c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 21 Apr 2022 10:16:43 +0800 Subject: [PATCH 47/47] stmt --- source/client/src/clientImpl.c | 3 +- source/client/src/clientStmt.c | 4 +++ tests/script/api/batchprepare.c | 55 +++++++++++++++++---------------- 3 files changed, 34 insertions(+), 28 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index d626fffc8e..5b3e8446c9 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -725,7 +725,8 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int int32_t len = taosUcs4ToMbs((TdUcs4*)varDataVal(pStart), varDataLen(pStart), varDataVal(p)); ASSERT(len <= bytes); - + ASSERT((p + len) < (pResultInfo->convertBuf[i] + colLength[i])); + varDataSetLen(p, len); pCol->offset[j] = (p - pResultInfo->convertBuf[i]); p += (len + VARSTR_HEADER_SIZE); diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index e6a47714d9..8a5134e02a 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -108,6 +108,10 @@ int32_t stmtParseSql(STscStmt* pStmt) { .setExecInfoFn = stmtSetExecInfo, .getExecInfoFn = stmtGetExecInfo, }; + + if (NULL == pStmt->exec.pRequest) { + STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest)); + } STMT_ERR_RET(parseSql(pStmt->exec.pRequest, false, &pStmt->sql.pQuery, &stmtCb)); diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 7fdcfcd167..2e046bd3ff 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -11,7 +11,7 @@ int32_t shortColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_INT}; int32_t fullColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_BOOL, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_UTINYINT, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_USMALLINT, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_UINT, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_UBIGINT, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_BINARY, TSDB_DATA_TYPE_NCHAR}; -int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_FLOAT}; +int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_NCHAR}; #define tListLen(x) (sizeof(x) / sizeof((x)[0])) @@ -70,7 +70,6 @@ typedef struct { } CaseCfg; CaseCfg gCase[] = { -#if 0 {"insert:MBSE1-FULL", tListLen(shortColList), shortColList, false, true, insertMBSETest1, 1, 10, 10, 0, 0, 1}, {"insert:MBSE1-FULL", tListLen(shortColList), shortColList, false, true, insertMBSETest1, 10, 100, 10, 0, 0, 1}, @@ -97,10 +96,10 @@ CaseCfg gCase[] = { {"insert:MBME4-FULL", tListLen(fullColList), fullColList, false, true, insertMBMETest4, 10, 10, 2, 0, 0, 1}, {"insert:MBME4-C012", tListLen(fullColList), fullColList, false, false, insertMBMETest4, 10, 10, 2, 12, 0, 1}, {"insert:MBME4-C002", tListLen(fullColList), fullColList, false, false, insertMBMETest4, 10, 10, 2, 2, 0, 1}, -#endif + {"insert:MPME1-FULL", tListLen(fullColList), fullColList, false, true, insertMPMETest1, 10, 10, 2, 0, 0, 1}, -// {"insert:MPME1-C012", tListLen(fullColList), fullColList, false, false, insertMPMETest1, 10, 10, 2, 12, 0, 1}, + {"insert:MPME1-C012", tListLen(fullColList), fullColList, false, false, insertMPMETest1, 10, 10, 2, 12, 0, 1}, }; @@ -111,6 +110,8 @@ typedef struct { bool autoCreate; bool checkParamNum; bool printRes; + bool printCreateTblSql; + bool printInsertSql; int32_t rowNum; //row num for one table int32_t bindColNum; int32_t bindRowNum; //row num for once bind @@ -122,6 +123,8 @@ typedef struct { CaseCtrl gCaseCtrl = { .bindNullNum = 0, .autoCreate = false, + .printCreateTblSql = false, + .printInsertSql = true, .rowNum = 0, .bindColNum = 0, .bindRowNum = 0, @@ -243,7 +246,10 @@ void generateInsertSQL(BindData *data) { len += sprintf(data->sql + len, "?"); } len += sprintf(data->sql + len, ")"); - + + if (gCaseCtrl.printInsertSql) { + printf("SQL: %s\n", data->sql); + } } void generateDataType(BindData *data, int32_t bindIdx, int32_t colIdx, int32_t *dataType) { @@ -525,8 +531,6 @@ int insertMBSETest1(TAOS_STMT *stmt) { BindData data = {0}; prepareData(&data); - printf("SQL: %s\n", data.sql); - int code = taos_stmt_prepare(stmt, data.sql, 0); if (code != 0){ printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); @@ -582,8 +586,6 @@ int insertMBSETest2(TAOS_STMT *stmt) { BindData data = {0}; prepareData(&data); - printf("SQL: %s\n", data.sql); - int code = taos_stmt_prepare(stmt, data.sql, 0); if (code != 0){ printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); @@ -638,8 +640,6 @@ int insertMBSETest2(TAOS_STMT *stmt) { int insertMBMETest1(TAOS_STMT *stmt) { BindData data = {0}; prepareData(&data); - - printf("SQL: %s\n", data.sql); int code = taos_stmt_prepare(stmt, data.sql, 0); if (code != 0){ @@ -695,8 +695,6 @@ int insertMBMETest2(TAOS_STMT *stmt) { BindData data = {0}; prepareData(&data); - printf("SQL: %s\n", data.sql); - int code = taos_stmt_prepare(stmt, data.sql, 0); if (code != 0){ printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); @@ -751,8 +749,6 @@ int insertMBMETest3(TAOS_STMT *stmt) { BindData data = {0}; prepareData(&data); - printf("SQL: %s\n", data.sql); - int code = taos_stmt_prepare(stmt, data.sql, 0); if (code != 0){ printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); @@ -818,8 +814,6 @@ int insertMBMETest4(TAOS_STMT *stmt) { BindData data = {0}; prepareData(&data); - printf("SQL: %s\n", data.sql); - int code = taos_stmt_prepare(stmt, data.sql, 0); if (code != 0){ printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); @@ -878,8 +872,6 @@ int insertMPMETest1(TAOS_STMT *stmt) { BindData data = {0}; prepareData(&data); - printf("SQL: %s\n", data.sql); - int code = taos_stmt_prepare(stmt, data.sql, 0); if (code != 0){ printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); @@ -3866,6 +3858,8 @@ void prepareCheckResult(TAOS *taos) { prepareCheckResultImpl(taos, buf, gCaseCtrl.printRes, gCurCase->rowNum * gExecLoopTimes); } + + gExecLoopTimes = 1; } @@ -4133,7 +4127,9 @@ void generateCreateTableSQL(char *buf, int32_t tblIdx, int32_t colNum, int32_t * blen += sprintf(buf + blen, ")"); - printf("Create SQL:%s\n", buf); + if (gCaseCtrl.printCreateTblSql) { + printf("Create Table SQL:%s\n", buf); + } } void prepare(TAOS *taos, int32_t colNum, int32_t *colList, int autoCreate) { @@ -4253,40 +4249,45 @@ void* runcase(TAOS *taos) { } void runAll(TAOS *taos) { -#if 0 + printf("Normal Test\n"); runcase(taos); + printf("Null Test\n"); gCaseCtrl.bindNullNum = 1; runcase(taos); gCaseCtrl.bindNullNum = 0; + printf("Bind Row Test\n"); gCaseCtrl.bindRowNum = 1; runcase(taos); gCaseCtrl.bindRowNum = 0; -#endif - gCaseCtrl.rowNum = 10000; + + printf("Row Num Test\n"); + gCaseCtrl.rowNum = 1000; gCaseCtrl.printRes = false; runcase(taos); gCaseCtrl.rowNum = 0; gCaseCtrl.printRes = true; -#if 0 - gCaseCtrl.runTimes = 10; + printf("Runtimes Test\n"); + gCaseCtrl.runTimes = 2; runcase(taos); gCaseCtrl.runTimes = 0; + printf("Check Param Test\n"); gCaseCtrl.checkParamNum = true; runcase(taos); gCaseCtrl.checkParamNum = false; + printf("Bind Col Num Test\n"); gCaseCtrl.bindColNum = 6; runcase(taos); gCaseCtrl.bindColNum = 0; + printf("Bind Col Type Test\n"); gCaseCtrl.bindColTypeNum = tListLen(bindColTypeList); - gCaseCtrl.bindColTypeNum = bindColTypeList = bindColTypeList; + gCaseCtrl.bindColTypeList = bindColTypeList; runcase(taos); -#endif } int main(int argc, char *argv[])