From 5b3087e48b5bc0627e7540c90252dbd6fb879fd1 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 19 Apr 2023 00:55:26 +0800 Subject: [PATCH 001/109] opti:escape logic in schemaless --- include/common/tcommon.h | 2 + source/client/inc/clientSml.h | 15 +++ source/client/src/clientSml.c | 25 +++-- source/client/src/clientSmlLine.c | 170 ++++++++++++++++++++---------- source/client/src/clientTmq.c | 6 +- 5 files changed, 151 insertions(+), 67 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index a97c68be49..b83d91d85d 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -341,6 +341,8 @@ typedef struct { float f; }; size_t length; + bool keyEscaped; + bool valueEscaped; } SSmlKv; #define QUERY_ASC_FORWARD_STEP 1 diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index 92896e6f23..bc342495be 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -107,6 +107,7 @@ typedef struct { int32_t colsLen; int32_t timestampLen; + bool measureEscaped; SArray *colArray; } SSmlLineInfo; @@ -206,6 +207,19 @@ typedef struct { #define IS_SAME_KEY (maxKV->keyLen == kv.keyLen && memcmp(maxKV->key, kv.key, kv.keyLen) == 0) +#define IS_SLASH_LETTER_IN_MEASUREMENT(sql) \ + (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE)) + +#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len)) + +#define PROCESS_SLASH_IN_MEASUREMENT(key, keyLen) \ + for (int i = 1; i < keyLen; ++i) { \ + if (IS_SLASH_LETTER_IN_MEASUREMENT(key + i)) { \ + MOVE_FORWARD_ONE(key + i, keyLen - i); \ + keyLen--; \ + } \ + } + extern int64_t smlFactorNS[3]; extern int64_t smlFactorS[3]; @@ -237,6 +251,7 @@ uint8_t smlGetTimestampLen(int64_t num); void clearColValArray(SArray* pCols); void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag); +void freeSSmlKv(void* data); int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); int32_t smlParseJSON(SSmlHandle *info, char *payload); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 17150286e1..855e48b88c 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -765,8 +765,12 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { size_t superTableLen = 0; void *superTable = taosHashGetKey(tmp, &superTableLen); + char* measure = taosMemoryMalloc(superTableLen); + memcpy(measure, superTable, superTableLen); + PROCESS_SLASH_IN_MEASUREMENT(measure, superTableLen); memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); - memcpy(pName.tname, superTable, superTableLen); + memcpy(pName.tname, measure, superTableLen); + taosMemoryFree(measure); code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); @@ -1049,7 +1053,7 @@ void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag) { // } // taosMemoryFree(tag->key); taosArrayDestroy(tag->cols); - taosArrayDestroy(tag->tags); + taosArrayDestroyEx(tag->tags, freeSSmlKv); taosMemoryFree(tag); } @@ -1063,6 +1067,12 @@ void clearColValArray(SArray *pCols) { } } +void freeSSmlKv(void* data){ + SSmlKv *kv = (SSmlKv*)data; + if(kv->keyEscaped) taosMemoryFree((void*)(kv->key)); + if(kv->valueEscaped) taosMemoryFree((void*)(kv->value)); +} + void smlDestroyInfo(SSmlHandle *info) { if (!info) return; qDestroyQuery(info->pQuery); @@ -1098,11 +1108,11 @@ void smlDestroyInfo(SSmlHandle *info) { } taosArrayDestroy(info->valueJsonArray); - taosArrayDestroy(info->preLineTagKV); + taosArrayDestroyEx(info->preLineTagKV, freeSSmlKv); if (!info->dataFormat) { for (int i = 0; i < info->lineNum; i++) { - taosArrayDestroy(info->lines[i].colArray); + taosArrayDestroyEx(info->lines[i].colArray, freeSSmlKv); if (info->parseJsonByLib) { taosMemoryFree(info->lines[i].tags); } @@ -1420,14 +1430,14 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char char cTmp = 0; // for print tmp if is raw if (info->isRawLine) { - cTmp = tmp[len - 1]; - tmp[len - 1] = '\0'; + cTmp = tmp[len]; + tmp[len] = '\0'; } uDebug("SML:0x%" PRIx64 " smlParseLine israw:%d, numLines:%d, protocol:%d, len:%d, sql:%s", info->id, info->isRawLine, numLines, info->protocol, len, tmp); if (info->isRawLine) { - tmp[len - 1] = cTmp; + tmp[len] = cTmp; } if (info->protocol == TSDB_SML_LINE_PROTOCOL) { @@ -1449,6 +1459,7 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE; } if (code != TSDB_CODE_SUCCESS) { + tmp[len] = '\0'; uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, tmp); return code; } diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index 335e3a1dc7..f4f4a5b630 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -21,32 +21,33 @@ #include "clientSml.h" // comma , -// #define IS_SLASH_COMMA(sql) (*(sql) == COMMA && *((sql)-1) == SLASH) #define IS_COMMA(sql) (*(sql) == COMMA && *((sql)-1) != SLASH) // space -// #define IS_SLASH_SPACE(sql) (*(sql) == SPACE && *((sql)-1) == SLASH) #define IS_SPACE(sql) (*(sql) == SPACE && *((sql)-1) != SLASH) // equal = -// #define IS_SLASH_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) == SLASH) #define IS_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) != SLASH) // quote " -// #define IS_SLASH_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) == SLASH) -#define IS_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) != SLASH) +//#define IS_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) != SLASH) // SLASH -// #define IS_SLASH_SLASH(sql) (*(sql) == SLASH && *((sql)-1) == SLASH) -#define IS_SLASH_LETTER(sql) \ - (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == EQUAL || *(sql) == QUOTE || \ - *(sql) == SLASH)) // (IS_SLASH_COMMA(sql) || IS_SLASH_SPACE(sql) || IS_SLASH_EQUAL(sql) || - // IS_SLASH_QUOTE(sql) || IS_SLASH_SLASH(sql)) +#define IS_SLASH_LETTER_IN_FIELD_VALUE(sql) \ + (*((sql)-1) == SLASH && (*(sql) == QUOTE || *(sql) == SLASH)) -#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len)) +#define IS_SLASH_LETTER_IN_TAG_FIELD_KEY(sql) \ + (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == EQUAL)) -#define PROCESS_SLASH(key, keyLen) \ - for (int i = 1; i < keyLen; ++i) { \ - if (IS_SLASH_LETTER(key + i)) { \ +#define PROCESS_SLASH_IN_FIELD_VALUE(key, keyLen) \ + for (int i = 1; i < keyLen; ++i) { \ + if (IS_SLASH_LETTER_IN_FIELD_VALUE(key + i)) { \ + MOVE_FORWARD_ONE(key + i, keyLen - i); \ + keyLen--; \ + } \ + } + +#define PROCESS_SLASH_IN_TAG_FIELD_KEY(key, keyLen) \ + for (int i = 1; i < keyLen; ++i) { \ + if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(key + i)) { \ MOVE_FORWARD_ONE(key + i, keyLen - i); \ - i--; \ keyLen--; \ } \ } @@ -151,7 +152,17 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin SSmlSTableMeta *sMeta = NULL; if (unlikely(tmp == NULL)) { - STableMeta *pTableMeta = smlGetMeta(info, currElement->measure, currElement->measureLen); + char* measure = currElement->measure; + int measureLen = currElement->measureLen; + if(currElement->measureEscaped){ + measure = taosMemoryMalloc(currElement->measureLen); + memcpy(measure, currElement->measure, currElement->measureLen); + PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); + } + STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen); + if(currElement->measureEscaped){ + taosMemoryFree(measure); + } if (pTableMeta == NULL) { info->dataFormat = false; info->reRun = true; @@ -171,17 +182,18 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin info->maxTagKVs = (*tmp)->tags; } } - taosArrayClear(preLineKV); + taosArrayClearEx(preLineKV, freeSSmlKv); while (*sql < sqlEnd) { if (unlikely(IS_SPACE(*sql))) { break; } - bool hasSlash = false; // parse key const char *key = *sql; size_t keyLen = 0; + bool keyEscaped = false; + size_t keyLenEscaped = 0; while (*sql < sqlEnd) { if (unlikely(IS_COMMA(*sql))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); @@ -192,16 +204,14 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin (*sql)++; break; } - if (!hasSlash) { - hasSlash = (*(*sql) == SLASH); + if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) { + keyLenEscaped++; + keyEscaped = true; } (*sql)++; } - if (unlikely(hasSlash)) { - PROCESS_SLASH(key, keyLen) - } - if (unlikely(IS_INVALID_COL_LEN(keyLen))) { + if (unlikely(IS_INVALID_COL_LEN(keyLen - keyLenEscaped))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; } @@ -209,7 +219,8 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin // parse value const char *value = *sql; size_t valueLen = 0; - hasSlash = false; + bool valueEscaped = false; + size_t valueLenEscaped = 0; while (*sql < sqlEnd) { // parse value if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) { @@ -219,8 +230,9 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin return TSDB_CODE_SML_INVALID_DATA; } - if (!hasSlash) { - hasSlash = (*(*sql) == SLASH); + if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(value)) { + valueLenEscaped++; + valueEscaped = true; } (*sql)++; @@ -232,15 +244,24 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin return TSDB_CODE_SML_INVALID_DATA; } - if (unlikely(hasSlash)) { - PROCESS_SLASH(value, valueLen) - } - - if (unlikely(valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) { + if (unlikely(valueLen - valueLenEscaped > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) { return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; } - SSmlKv kv = {.key = key, .keyLen = keyLen, .type = TSDB_DATA_TYPE_NCHAR, .value = value, .length = valueLen}; + if (keyEscaped){ + char *tmp = (char*)taosMemoryMalloc(keyLen); + memcpy(tmp, key, keyLen); + PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, keyLen); + key = tmp; + } + if (valueEscaped){ + char *tmp = (char*)taosMemoryMalloc(valueLen); + memcpy(tmp, value, valueLen); + PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, valueLen); + value = tmp; + } + SSmlKv kv = {.key = key, .keyLen = keyLen, .type = TSDB_DATA_TYPE_NCHAR, .value = value, .length = valueLen, .keyEscaped = keyEscaped, .valueEscaped = valueEscaped}; + taosArrayPush(preLineKV, &kv); if (info->dataFormat) { if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) { info->dataFormat = false; @@ -266,7 +287,6 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin info->needModifySchema = true; } } - taosArrayPush(preLineKV, &kv); cnt++; if (IS_SPACE(*sql)) { @@ -285,6 +305,11 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin return TSDB_CODE_OUT_OF_MEMORY; } tinfo->tags = taosArrayDup(preLineKV, NULL); + for(size_t i = 0; i < taosArrayGetSize(preLineKV); i++){ + SSmlKv *kv = (SSmlKv *)taosArrayGet(preLineKV, i); + if(kv->keyEscaped)kv->key = NULL; + if(kv->valueEscaped)kv->value = NULL; + } smlSetCTableName(tinfo); tinfo->uid = info->uid++; @@ -321,7 +346,17 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen); if (unlikely(tmp == NULL)) { - STableMeta *pTableMeta = smlGetMeta(info, currElement->measure, currElement->measureLen); + char* measure = currElement->measure; + int measureLen = currElement->measureLen; + if(currElement->measureEscaped){ + measure = taosMemoryMalloc(currElement->measureLen); + memcpy(measure, currElement->measure, currElement->measureLen); + PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); + } + STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen); + if(currElement->measureEscaped){ + taosMemoryFree(measure); + } if (pTableMeta == NULL) { info->dataFormat = false; info->reRun = true; @@ -352,10 +387,11 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin break; } - bool hasSlash = false; // parse key const char *key = *sql; size_t keyLen = 0; + bool keyEscaped = false; + size_t keyLenEscaped = 0; while (*sql < sqlEnd) { if (unlikely(IS_COMMA(*sql))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); @@ -366,16 +402,14 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin (*sql)++; break; } - if (!hasSlash) { - hasSlash = (*(*sql) == SLASH); + if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) { + keyLenEscaped++; + keyEscaped = true; } (*sql)++; } - if (unlikely(hasSlash)) { - PROCESS_SLASH(key, keyLen) - } - if (unlikely(IS_INVALID_COL_LEN(keyLen))) { + if (unlikely(IS_INVALID_COL_LEN(keyLen - keyLenEscaped))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; } @@ -383,11 +417,13 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin // parse value const char *value = *sql; size_t valueLen = 0; - hasSlash = false; - bool isInQuote = false; + bool valueEscaped = false; + size_t valueLenEscaped = 0; + bool isInQuote = false; + const char *escapeChar = NULL; while (*sql < sqlEnd) { // parse value - if (unlikely(IS_QUOTE(*sql))) { + if (unlikely(*(*sql) == QUOTE && (*(*sql - 1) != SLASH || (*sql - 1) == escapeChar))) { isInQuote = !isInQuote; (*sql)++; continue; @@ -395,13 +431,12 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin if (!isInQuote) { if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) { break; - } else if (unlikely(IS_EQUAL(*sql))) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); - return TSDB_CODE_SML_INVALID_DATA; } } - if (!hasSlash) { - hasSlash = (*(*sql) == SLASH); + if (IS_SLASH_LETTER_IN_FIELD_VALUE(*sql) && (*sql - 1) != escapeChar) { + escapeChar = *sql; + valueEscaped = true; + valueLenEscaped++; } (*sql)++; @@ -416,14 +451,25 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); return TSDB_CODE_SML_INVALID_DATA; } - if (unlikely(hasSlash)) { - PROCESS_SLASH(value, valueLen) + + if (keyEscaped){ + char *tmp = (char*)taosMemoryMalloc(keyLen); + memcpy(tmp, key, keyLen); + PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, keyLen); + key = tmp; + } + if (valueEscaped){ + char *tmp = (char*)taosMemoryMalloc(valueLen); + memcpy(tmp, value, valueLen); + PROCESS_SLASH_IN_FIELD_VALUE(tmp, valueLen); + value = tmp; } SSmlKv kv = {.key = key, .keyLen = keyLen, .value = value, .length = valueLen}; int32_t ret = smlParseValue(&kv, &info->msgBuf); if (ret != TSDB_CODE_SUCCESS) { smlBuildInvalidDataMsg(&info->msgBuf, "smlParseValue error", value); + freeSSmlKv(&kv); return ret; } @@ -432,6 +478,7 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin if (unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)) { info->dataFormat = false; info->reRun = true; + freeSSmlKv(&kv); return TSDB_CODE_SUCCESS; } // bind data @@ -440,22 +487,26 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin uDebug("smlBuildCol error, retry"); info->dataFormat = false; info->reRun = true; + freeSSmlKv(&kv); return TSDB_CODE_SUCCESS; } if (cnt >= taosArrayGetSize(info->masColKVs)) { info->dataFormat = false; info->reRun = true; + freeSSmlKv(&kv); return TSDB_CODE_SUCCESS; } SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->masColKVs, cnt); if (kv.type != maxKV->type) { info->dataFormat = false; info->reRun = true; + freeSSmlKv(&kv); return TSDB_CODE_SUCCESS; } if (unlikely(!IS_SAME_KEY)) { info->dataFormat = false; info->reRun = true; + freeSSmlKv(&kv); return TSDB_CODE_SUCCESS; } @@ -463,6 +514,7 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin maxKV->length = kv.length; info->needModifySchema = true; } + freeSSmlKv(&kv); } else { if (currElement->colArray == NULL) { currElement->colArray = taosArrayInit_s(sizeof(SSmlKv), 1); @@ -487,10 +539,12 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine elements->measure = sql; // parse measure + size_t measureLenEscaped = 0; while (sql < sqlEnd) { - if (unlikely((sql != elements->measure) && IS_SLASH_LETTER(sql))) { - MOVE_FORWARD_ONE(sql, sqlEnd - sql); - sqlEnd--; + if (unlikely((sql != elements->measure) && IS_SLASH_LETTER_IN_MEASUREMENT(sql))) { + elements->measureEscaped = true; + measureLenEscaped++; + sql++; continue; } if (unlikely(IS_COMMA(sql))) { @@ -503,7 +557,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine sql++; } elements->measureLen = sql - elements->measure; - if (unlikely(IS_INVALID_TABLE_LEN(elements->measureLen))) { + if (unlikely(IS_INVALID_TABLE_LEN(elements->measureLen - measureLenEscaped))) { smlBuildInvalidDataMsg(&info->msgBuf, "measure is empty or too large than 192", NULL); return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; } @@ -581,7 +635,9 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine .keyLen = TS_LEN, .type = TSDB_DATA_TYPE_TIMESTAMP, .i = ts, - .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; + .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, + .keyEscaped = false, + .valueEscaped = false}; if (info->dataFormat) { uDebug("SML:0x%" PRIx64 " smlParseInfluxString format true, ts:%" PRId64, info->id, ts); ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 0); diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index ceca06e309..6642e716e6 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1243,9 +1243,6 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { taosMemoryFree(pParam); if (code != 0) { - tscWarn("consumer:0x%" PRIx64 " msg from vgId:%d discarded, epoch %d, since %s, reqId:0x%" PRIx64, tmq->consumerId, - vgId, epoch, tstrerror(code), requestId); - if (pMsg->pData) taosMemoryFree(pMsg->pData); if (pMsg->pEpSet) taosMemoryFree(pMsg->pEpSet); @@ -1267,6 +1264,9 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { taosWriteQitem(tmq->mqueue, pRspWrapper); } else if (code == TSDB_CODE_WAL_LOG_NOT_EXIST) { // poll data while insert taosMsleep(500); + } else{ + tscError("consumer:0x%" PRIx64 " msg from vgId:%d discarded, epoch %d, since %s, reqId:0x%" PRIx64, tmq->consumerId, + vgId, epoch, tstrerror(code), requestId); } goto CREATE_MSG_FAIL; From aa8d25f6e548eb2e67d424c82aac38a9ef6b8d2a Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 19 Apr 2023 10:03:06 +0800 Subject: [PATCH 002/109] feat: support fetching table tag value --- include/libs/catalog/catalog.h | 1 + source/libs/catalog/inc/catalogInt.h | 8 ++ source/libs/catalog/src/ctgAsync.c | 127 ++++++++++++++++++++++++++- 3 files changed, 135 insertions(+), 1 deletion(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 2c684f8f76..2f8e7846f3 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -82,6 +82,7 @@ typedef struct SCatalogReq { SArray* pUser; // element is SUserAuthInfo SArray* pTableIndex; // element is SNAME SArray* pTableCfg; // element is SNAME + SArray* pTableTag; // element is SNAME bool qNodeRequired; // valid qnode bool dNodeRequired; // valid dnode bool svrVerRequired; diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 85a130d293..f0e5024c59 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -58,6 +58,7 @@ typedef enum { CTG_CI_OTHERTABLE_META, CTG_CI_TBL_SMA, CTG_CI_TBL_CFG, + CTG_CI_TBL_TAG, CTG_CI_INDEX_INFO, CTG_CI_USER, CTG_CI_UDF, @@ -110,6 +111,7 @@ typedef enum { CTG_TASK_GET_SVR_VER, CTG_TASK_GET_TB_META_BATCH, CTG_TASK_GET_TB_HASH_BATCH, + CTG_TASK_GET_TB_TAG, } CTG_TASK_TYPE; typedef enum { @@ -186,6 +188,11 @@ typedef struct SCtgTbCfgCtx { SVgroupInfo* pVgInfo; } SCtgTbCfgCtx; +typedef struct SCtgTbTagCtx { + SName* pName; + SVgroupInfo* pVgInfo; +} SCtgTbTagCtx; + typedef struct SCtgDbVgCtx { char dbFName[TSDB_DB_FNAME_LEN]; } SCtgDbVgCtx; @@ -304,6 +311,7 @@ typedef struct SCtgJob { catalogCallback userFp; int32_t tbMetaNum; int32_t tbHashNum; + int32_t tbTagNum; int32_t dbVgNum; int32_t udfNum; int32_t qnodeNum; diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index f2a354997d..b10da5cc73 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -386,6 +386,37 @@ int32_t ctgInitGetTbCfgTask(SCtgJob* pJob, int32_t taskIdx, void* param) { return TSDB_CODE_SUCCESS; } +int32_t ctgInitGetTbTagTask(SCtgJob* pJob, int32_t taskIdx, void* param) { + SName* name = (SName*)param; + SCtgTask task = {0}; + + task.type = CTG_TASK_GET_TB_TAG; + task.taskId = taskIdx; + task.pJob = pJob; + + task.taskCtx = taosMemoryCalloc(1, sizeof(SCtgTbTagCtx)); + if (NULL == task.taskCtx) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + SCtgTbTagCtx* ctx = task.taskCtx; + ctx->pName = taosMemoryMalloc(sizeof(*name)); + if (NULL == ctx->pName) { + taosMemoryFree(task.taskCtx); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + memcpy(ctx->pName, name, sizeof(*name)); + + taosArrayPush(pJob->pTasks, &task); + + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, + ctgTaskTypeStr(task.type), name->tname); + + return TSDB_CODE_SUCCESS; +} + + int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob* pJob, const SCatalogReq* pReq) { SHashObj* pDb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); SHashObj* pTb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); @@ -437,6 +468,15 @@ int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob* pJob, con char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(name, dbFName); taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); + taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); + } + + for (int32_t i = 0; i < pJob->tbTagNum; ++i) { + SName* name = taosArrayGet(pReq->pTableTag, i); + char dbFName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(name, dbFName); + taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); + taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); } char* dbFName = taosHashIterate(pDb, NULL); @@ -505,9 +545,10 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const int32_t dbInfoNum = (int32_t)taosArrayGetSize(pReq->pDbInfo); int32_t tbIndexNum = (int32_t)taosArrayGetSize(pReq->pTableIndex); int32_t tbCfgNum = (int32_t)taosArrayGetSize(pReq->pTableCfg); + int32_t tbTagNum = (int32_t)ctgGetTablesReqNum(pReq->pTableTag); int32_t taskNum = tbMetaNum + dbVgNum + udfNum + tbHashNum + qnodeNum + dnodeNum + svrVerNum + dbCfgNum + indexNum + - userNum + dbInfoNum + tbIndexNum + tbCfgNum; + userNum + dbInfoNum + tbIndexNum + tbCfgNum + tbTagNum; *job = taosMemoryCalloc(1, sizeof(SCtgJob)); if (NULL == *job) { @@ -537,6 +578,7 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const pJob->tbIndexNum = tbIndexNum; pJob->tbCfgNum = tbCfgNum; pJob->svrVerNum = svrVerNum; + pJob->tbTagNum = tbTagNum; #if CTG_BATCH_FETCH pJob->pBatchs = @@ -604,6 +646,12 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_CFG, name, NULL)); } + for (int32_t i = 0; i < tbCfgNum; ++i) { + SName* name = taosArrayGet(pReq->pTableTag, i); + CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_TAG, name, NULL)); + } + + for (int32_t i = 0; i < indexNum; ++i) { char* indexName = taosArrayGet(pReq->pIndex, i); CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_INDEX_INFO, indexName, NULL)); @@ -1473,6 +1521,24 @@ _return: CTG_RET(code); } + +int32_t ctgHandleGetTbTagRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { + int32_t code = 0; + SCtgTask* pTask = tReq->pTask; + CTG_ERR_JRET(ctgProcessRspMsg(&pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); + + STableCfgRsp* pRsp = (STableCfgRsp*)pTask->msgCtx.out; + + TSWAP(pTask->res, pTask->msgCtx.out); + +_return: + + ctgHandleTaskEnd(pTask, code); + + CTG_RET(code); +} + + int32_t ctgHandleGetDbCfgRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; @@ -1935,6 +2001,45 @@ _return: CTG_RET(code); } + +int32_t ctgLaunchGetTbTagTask(SCtgTask* pTask) { + int32_t code = 0; + SCatalog* pCtg = pTask->pJob->pCtg; + SRequestConnInfo* pConn = &pTask->pJob->conn; + SCtgTbTagCtx* pCtx = (SCtgTbTagCtx*)pTask->taskCtx; + SArray* pRes = NULL; + char dbFName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(pCtx->pName, dbFName); + SCtgJob* pJob = pTask->pJob; + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx->pBatchs) { + pMsgCtx->pBatchs = pJob->pBatchs; + } + + if (NULL == pCtx->pVgInfo) { + CTG_ERR_JRET(ctgGetTbHashVgroupFromCache(pCtg, pCtx->pName, &pCtx->pVgInfo)); + if (NULL == pCtx->pVgInfo) { + CTG_ERR_JRET(ctgLaunchSubTask(pTask, CTG_TASK_GET_DB_VGROUP, ctgGetTbCfgCb, dbFName)); + return TSDB_CODE_SUCCESS; + } + } + + CTG_CACHE_NHIT_INC(CTG_CI_TBL_TAG, 1); + + CTG_ERR_JRET(ctgGetTableCfgFromVnode(pCtg, pConn, pCtx->pName, pCtx->pVgInfo, NULL, pTask)); + + return TSDB_CODE_SUCCESS; + +_return: + + if (CTG_TASK_LAUNCHED == pTask->status) { + ctgHandleTaskEnd(pTask, code); + } + + CTG_RET(code); +} + + int32_t ctgLaunchGetQnodeTask(SCtgTask* pTask) { SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; @@ -2138,6 +2243,25 @@ _return: CTG_RET(ctgHandleTaskEnd(pTask, pTask->subRes.code)); } +int32_t ctgGetTbTagCb(SCtgTask* pTask) { + int32_t code = 0; + + CTG_ERR_JRET(pTask->subRes.code); + + SCtgTbTagCtx* pCtx = (SCtgTbTagCtx*)pTask->taskCtx; + SDBVgInfo* pDb = (SDBVgInfo*)pTask->subRes.res; + + pCtx->pVgInfo = taosMemoryCalloc(1, sizeof(SVgroupInfo)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pTask->pJob->pCtg, pDb, pCtx->pName, pCtx->pVgInfo)); + + CTG_RET(ctgLaunchGetTbTagTask(pTask)); + +_return: + + CTG_RET(ctgHandleTaskEnd(pTask, pTask->subRes.code)); +} + + int32_t ctgGetUserCb(SCtgTask* pTask) { int32_t code = 0; @@ -2197,6 +2321,7 @@ SCtgAsyncFps gCtgAsyncFps[] = { {ctgInitGetSvrVerTask, ctgLaunchGetSvrVerTask, ctgHandleGetSvrVerRsp, ctgDumpSvrVer, NULL, NULL}, {ctgInitGetTbMetasTask, ctgLaunchGetTbMetasTask, ctgHandleGetTbMetasRsp, ctgDumpTbMetasRes, NULL, NULL}, {ctgInitGetTbHashsTask, ctgLaunchGetTbHashsTask, ctgHandleGetTbHashsRsp, ctgDumpTbHashsRes, NULL, NULL}, + {ctgInitGetTbTagTask, ctgLaunchGetTbTagTask, ctgHandleGetTbTagRsp, ctgDumpTbTagRes, NULL, NULL}, }; int32_t ctgMakeAsyncRes(SCtgJob* pJob) { From 742b5ee08c6f395488ba978eb00e51acc086b541 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 20 Apr 2023 13:42:16 +0800 Subject: [PATCH 003/109] other: update wal logs level. --- source/dnode/vnode/src/tq/tq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 8f26d5868c..962b89732e 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1309,7 +1309,7 @@ int32_t tqStartStreamTasks(STQ* pTq) { return -1; } - tqInfo("vgId:%d start wal scan stream tasks, tasks:%d", vgId, numOfTasks); + tqDebug("vgId:%d start wal scan stream tasks, tasks:%d", vgId, numOfTasks); initOffsetForAllRestoreTasks(pTq); pRunReq->head.vgId = vgId; From ba0a6e087f8feb7de4ca6d1077bec691f13f848c Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 20 Apr 2023 13:51:39 +0800 Subject: [PATCH 004/109] test: add the cast that select field include two udf function --- source/libs/function/CMakeLists.txt | 20 +++++++++++ source/libs/function/test/udf1_dup.c | 42 ++++++++++++++++++++++++ tests/system-test/0-others/udf_create.py | 12 +++++++ 3 files changed, 74 insertions(+) create mode 100644 source/libs/function/test/udf1_dup.c diff --git a/source/libs/function/CMakeLists.txt b/source/libs/function/CMakeLists.txt index 9d11d7b376..147f697302 100644 --- a/source/libs/function/CMakeLists.txt +++ b/source/libs/function/CMakeLists.txt @@ -79,6 +79,26 @@ ENDIF () target_link_libraries( udf1 PUBLIC os ${LINK_JEMALLOC}) + +add_library(udf1_dup STATIC MODULE test/udf1_dup.c) +target_include_directories( + udf1_dup + PUBLIC + "${TD_SOURCE_DIR}/include/libs/function" + "${TD_SOURCE_DIR}/include/util" + "${TD_SOURCE_DIR}/include/common" + "${TD_SOURCE_DIR}/include/client" + "${TD_SOURCE_DIR}/include/os" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +IF (TD_LINUX_64 AND JEMALLOC_ENABLED) + ADD_DEPENDENCIES(udf1_dup jemalloc) +ENDIF () + +target_link_libraries( + udf1_dup PUBLIC os ${LINK_JEMALLOC}) + add_library(udf2 STATIC MODULE test/udf2.c) target_include_directories( udf2 diff --git a/source/libs/function/test/udf1_dup.c b/source/libs/function/test/udf1_dup.c new file mode 100644 index 0000000000..71d7ebc58a --- /dev/null +++ b/source/libs/function/test/udf1_dup.c @@ -0,0 +1,42 @@ +#include +#include +#include +#ifdef LINUX +#include +#endif +#ifdef WINDOWS +#include +#endif +#include "taosudf.h" + + +DLL_EXPORT int32_t udf1_init() { return 0; } + +DLL_EXPORT int32_t udf1_destroy() { return 0; } + +DLL_EXPORT int32_t udf1(SUdfDataBlock *block, SUdfColumn *resultCol) { + SUdfColumnData *resultData = &resultCol->colData; + for (int32_t i = 0; i < block->numOfRows; ++i) { + int j = 0; + for (; j < block->numOfCols; ++j) { + if (udfColDataIsNull(block->udfCols[j], i)) { + udfColDataSetNull(resultCol, i); + break; + } + } + if (j == block->numOfCols) { + int32_t luckyNum = 2; + udfColDataSet(resultCol, i, (char *)&luckyNum, false); + } + } + // to simulate actual processing delay by udf +#ifdef LINUX + usleep(1 * 1000); // usleep takes sleep time in us (1 millionth of a second) +#endif +#ifdef WINDOWS + Sleep(1); +#endif + resultData->numOfRows = block->numOfRows; + return 0; +} + diff --git a/tests/system-test/0-others/udf_create.py b/tests/system-test/0-others/udf_create.py index f467e802ac..398d746a9b 100644 --- a/tests/system-test/0-others/udf_create.py +++ b/tests/system-test/0-others/udf_create.py @@ -47,14 +47,18 @@ class TDTestCase: if platform.system().lower() == 'windows': self.libudf1 = subprocess.Popen('(for /r %s %%i in ("udf1.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf1_dup = subprocess.Popen('(for /r %s %%i in ("udf1_dup.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") self.libudf2 = subprocess.Popen('(for /r %s %%i in ("udf2.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") if (not tdDnodes.dnodes[0].remoteIP == ""): tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf1.so',projPath+"\\debug\\build\\lib\\") + tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf1_dup.so',projPath+"\\debug\\build\\lib\\") tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf2.so',projPath+"\\debug\\build\\lib\\") self.libudf1 = self.libudf1.replace('udf1.dll','libudf1.so') + self.libudf1 = self.libudf1.replace('udf1.dll','libudf1_dup.so') self.libudf2 = self.libudf2.replace('udf2.dll','libudf2.so') else: self.libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf1_dup = subprocess.Popen('find %s -name "libudf1_dup.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") self.libudf2 = subprocess.Popen('find %s -name "libudf2.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") self.libudf1 = self.libudf1.replace('\r','').replace('\n','') self.libudf2 = self.libudf2.replace('\r','').replace('\n','') @@ -174,6 +178,7 @@ class TDTestCase: # create scalar functions tdSql.execute("create function udf1 as '%s' outputtype int;"%self.libudf1) + tdSql.execute("create function udf1_dup as '%s' outputtype int;"%self.libudf1_dup) # create aggregate functions @@ -188,6 +193,13 @@ class TDTestCase: # scalar functions + # udf1_dup + tdSql.query("select num1 , udf1(num1) ,udf1_dup(num1) from tb") + tdSql.checkData(1,0,1) + tdSql.checkData(1,1,2) + tdSql.checkData(2,0,1) + tdSql.checkData(2,1,2) + tdSql.execute("use db ") tdSql.query("select num1 , udf1(num1) ,num2 ,udf1(num2),num3 ,udf1(num3),num4 ,udf1(num4) from tb") tdSql.checkData(0,0,None) From cc78a6356c44b85e406a6e6272357034dd8ffa5d Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 20 Apr 2023 13:54:22 +0800 Subject: [PATCH 005/109] fix: do not performace table count scan optimized where there are no agg functions --- source/libs/planner/src/planOptimizer.c | 4 +++- tests/script/tsim/query/tableCount.sim | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 52bb03466c..e100a67b93 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2530,7 +2530,9 @@ static bool tbCntScanOptIsEligibleAggFuncs(SNodeList* pAggFuncs) { } static bool tbCntScanOptIsEligibleAgg(SAggLogicNode* pAgg) { - return tbCntScanOptIsEligibleGroupKeys(pAgg->pGroupKeys) && tbCntScanOptIsEligibleAggFuncs(pAgg->pAggFuncs); + return tbCntScanOptIsEligibleGroupKeys(pAgg->pGroupKeys) && + (NULL != pAgg->pAggFuncs) && + tbCntScanOptIsEligibleAggFuncs(pAgg->pAggFuncs); } static bool tbCntScanOptGetColValFromCond(SOperatorNode* pOper, SColumnNode** pCol, SValueNode** pVal) { diff --git a/tests/script/tsim/query/tableCount.sim b/tests/script/tsim/query/tableCount.sim index d8d9bb9b03..ac5e23273a 100644 --- a/tests/script/tsim/query/tableCount.sim +++ b/tests/script/tsim/query/tableCount.sim @@ -104,4 +104,9 @@ if $data62 != 5 then return -1 endi +sql select distinct db_name from information_schema.ins_tables; +print $rows +if $rows != 4 then + return -1 +endi system sh/exec.sh -n dnode1 -s stop -x SIGINT From 977af4289b3a942fa11a3b0d282f7627317ae595 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 20 Apr 2023 14:05:15 +0800 Subject: [PATCH 006/109] test: fix udf1_dup error --- source/libs/function/test/udf1_dup.c | 6 +++--- tests/system-test/0-others/udf_create.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/source/libs/function/test/udf1_dup.c b/source/libs/function/test/udf1_dup.c index 71d7ebc58a..c251192da3 100644 --- a/source/libs/function/test/udf1_dup.c +++ b/source/libs/function/test/udf1_dup.c @@ -10,11 +10,11 @@ #include "taosudf.h" -DLL_EXPORT int32_t udf1_init() { return 0; } +DLL_EXPORT int32_t udf1_dup_init() { return 0; } -DLL_EXPORT int32_t udf1_destroy() { return 0; } +DLL_EXPORT int32_t udf1_dup_destroy() { return 0; } -DLL_EXPORT int32_t udf1(SUdfDataBlock *block, SUdfColumn *resultCol) { +DLL_EXPORT int32_t udf1_dup(SUdfDataBlock *block, SUdfColumn *resultCol) { SUdfColumnData *resultData = &resultCol->colData; for (int32_t i = 0; i < block->numOfRows; ++i) { int j = 0; diff --git a/tests/system-test/0-others/udf_create.py b/tests/system-test/0-others/udf_create.py index 398d746a9b..e53289e92e 100644 --- a/tests/system-test/0-others/udf_create.py +++ b/tests/system-test/0-others/udf_create.py @@ -54,13 +54,14 @@ class TDTestCase: tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf1_dup.so',projPath+"\\debug\\build\\lib\\") tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf2.so',projPath+"\\debug\\build\\lib\\") self.libudf1 = self.libudf1.replace('udf1.dll','libudf1.so') - self.libudf1 = self.libudf1.replace('udf1.dll','libudf1_dup.so') + self.libudf1_dup = self.libudf1_dup.replace('udf1_dup.dll','libudf1_dup.so') self.libudf2 = self.libudf2.replace('udf2.dll','libudf2.so') else: self.libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") self.libudf1_dup = subprocess.Popen('find %s -name "libudf1_dup.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") self.libudf2 = subprocess.Popen('find %s -name "libudf2.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") self.libudf1 = self.libudf1.replace('\r','').replace('\n','') + self.libudf1_dup = self.libudf1_dup.replace('\r','').replace('\n','') self.libudf2 = self.libudf2.replace('\r','').replace('\n','') From de76a02acf1bcfa77be11ef64ed1ef4350ac24c3 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 20 Apr 2023 14:08:38 +0800 Subject: [PATCH 007/109] test: select fields error --- tests/system-test/0-others/udf_create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/0-others/udf_create.py b/tests/system-test/0-others/udf_create.py index e53289e92e..3e4ca86e52 100644 --- a/tests/system-test/0-others/udf_create.py +++ b/tests/system-test/0-others/udf_create.py @@ -195,7 +195,7 @@ class TDTestCase: # scalar functions # udf1_dup - tdSql.query("select num1 , udf1(num1) ,udf1_dup(num1) from tb") + tdSql.query("select udf1(num1) ,udf1_dup(num1) from tb") tdSql.checkData(1,0,1) tdSql.checkData(1,1,2) tdSql.checkData(2,0,1) From c34c53539d91a36c8c2d40e021581d58b02248e2 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 20 Apr 2023 14:32:32 +0800 Subject: [PATCH 008/109] test: add udf2_dup test case --- source/libs/function/CMakeLists.txt | 20 ++++++++++++++++++++ tests/system-test/0-others/udf_create.py | 10 ++++++++++ 2 files changed, 30 insertions(+) diff --git a/source/libs/function/CMakeLists.txt b/source/libs/function/CMakeLists.txt index 147f697302..f23b4d3e87 100644 --- a/source/libs/function/CMakeLists.txt +++ b/source/libs/function/CMakeLists.txt @@ -119,6 +119,26 @@ target_link_libraries( udf2 PUBLIC os ${LINK_JEMALLOC} ) +add_library(udf2_dup STATIC MODULE test/udf2_dup.c) +target_include_directories( + udf2_dup + PUBLIC + "${TD_SOURCE_DIR}/include/libs/function" + "${TD_SOURCE_DIR}/include/util" + "${TD_SOURCE_DIR}/include/common" + "${TD_SOURCE_DIR}/include/client" + "${TD_SOURCE_DIR}/include/os" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +IF (TD_LINUX_64 AND JEMALLOC_ENABLED) + ADD_DEPENDENCIES(udf2_dup jemalloc) +ENDIF () + +target_link_libraries( + udf2_dup PUBLIC os ${LINK_JEMALLOC} +) + #SET(EXECUTABLE_OUTPUT_PATH ${CMAKE_BINARY_DIR}/build/bin) add_executable(udfd src/udfd.c) target_include_directories( diff --git a/tests/system-test/0-others/udf_create.py b/tests/system-test/0-others/udf_create.py index 3e4ca86e52..ec7716fba3 100644 --- a/tests/system-test/0-others/udf_create.py +++ b/tests/system-test/0-others/udf_create.py @@ -49,20 +49,25 @@ class TDTestCase: self.libudf1 = subprocess.Popen('(for /r %s %%i in ("udf1.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") self.libudf1_dup = subprocess.Popen('(for /r %s %%i in ("udf1_dup.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") self.libudf2 = subprocess.Popen('(for /r %s %%i in ("udf2.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf2_dup = subprocess.Popen('(for /r %s %%i in ("udf2_dup.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") if (not tdDnodes.dnodes[0].remoteIP == ""): tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf1.so',projPath+"\\debug\\build\\lib\\") tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf1_dup.so',projPath+"\\debug\\build\\lib\\") tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf2.so',projPath+"\\debug\\build\\lib\\") + tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf2_dup.so',projPath+"\\debug\\build\\lib\\") self.libudf1 = self.libudf1.replace('udf1.dll','libudf1.so') self.libudf1_dup = self.libudf1_dup.replace('udf1_dup.dll','libudf1_dup.so') self.libudf2 = self.libudf2.replace('udf2.dll','libudf2.so') + self.libudf2_dup = self.libudf2_dup.replace('udf2_dup.dll','libudf2_dup.so') else: self.libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") self.libudf1_dup = subprocess.Popen('find %s -name "libudf1_dup.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") self.libudf2 = subprocess.Popen('find %s -name "libudf2.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf2_dup = subprocess.Popen('find %s -name "libudf2_dup.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") self.libudf1 = self.libudf1.replace('\r','').replace('\n','') self.libudf1_dup = self.libudf1_dup.replace('\r','').replace('\n','') self.libudf2 = self.libudf2.replace('\r','').replace('\n','') + self.libudf2_dup = self.libudf2_dup.replace('\r','').replace('\n','') def prepare_data(self): @@ -184,6 +189,7 @@ class TDTestCase: # create aggregate functions tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2) + tdSql.execute("create aggregate function udf2_dup as '%s' outputtype double bufSize 8;"%self.libudf2_dup) functions = tdSql.getResult("show functions") function_nums = len(functions) @@ -251,6 +257,10 @@ class TDTestCase: # aggregate functions + tdSql.query("select udf2(num1) ,udf2_dup(num2) from tb") + val = tdSql.queryResult[0][0] + 100 + tdSql.checkData(0,1,val) + tdSql.query("select udf2(num1) ,udf2(num2), udf2(num3) from tb") tdSql.checkData(0,0,15.362291496) tdSql.checkData(0,1,10000949.553189287) From 4c0cdd8192c9e0126723087823efb945a7c6ac0e Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 20 Apr 2023 14:34:38 +0800 Subject: [PATCH 009/109] test: add udf2_dup test case --- source/libs/function/test/udf2_dup.c | 78 ++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 source/libs/function/test/udf2_dup.c diff --git a/source/libs/function/test/udf2_dup.c b/source/libs/function/test/udf2_dup.c new file mode 100644 index 0000000000..1a98190823 --- /dev/null +++ b/source/libs/function/test/udf2_dup.c @@ -0,0 +1,78 @@ +#include +#include +#include +#include + +#include "taosudf.h" + +DLL_EXPORT int32_t udf2_dup_init() { return 0; } + +DLL_EXPORT int32_t udf2_dup_destroy() { return 0; } + +DLL_EXPORT int32_t udf2_dup_start(SUdfInterBuf* buf) { + *(int64_t*)(buf->buf) = 0; + buf->bufLen = sizeof(double); + buf->numOfResult = 1; + return 0; +} + +DLL_EXPORT int32_t udf2_dup(SUdfDataBlock* block, SUdfInterBuf* interBuf, SUdfInterBuf* newInterBuf) { + double sumSquares = 0; + if (interBuf->numOfResult == 1) { + sumSquares = *(double*)interBuf->buf; + } + int8_t numNotNull = 0; + for (int32_t i = 0; i < block->numOfCols; ++i) { + SUdfColumn* col = block->udfCols[i]; + if (!(col->colMeta.type == TSDB_DATA_TYPE_INT || col->colMeta.type == TSDB_DATA_TYPE_DOUBLE)) { + return TSDB_CODE_UDF_INVALID_INPUT; + } + } + for (int32_t i = 0; i < block->numOfCols; ++i) { + for (int32_t j = 0; j < block->numOfRows; ++j) { + SUdfColumn* col = block->udfCols[i]; + if (udfColDataIsNull(col, j)) { + continue; + } + switch (col->colMeta.type) { + case TSDB_DATA_TYPE_INT: { + char* cell = udfColDataGetData(col, j); + int32_t num = *(int32_t*)cell; + sumSquares += (double)num * num; + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + char* cell = udfColDataGetData(col, j); + double num = *(double*)cell; + sumSquares += num * num; + break; + } + default: + break; + } + ++numNotNull; + } + } + + *(double*)(newInterBuf->buf) = sumSquares; + newInterBuf->bufLen = sizeof(double); + + if (interBuf->numOfResult == 0 && numNotNull == 0) { + newInterBuf->numOfResult = 0; + } else { + newInterBuf->numOfResult = 1; + } + return 0; +} + +DLL_EXPORT int32_t udf2_dup_finish(SUdfInterBuf* buf, SUdfInterBuf* resultData) { + if (buf->numOfResult == 0) { + resultData->numOfResult = 0; + return 0; + } + double sumSquares = *(double*)(buf->buf); + *(double*)(resultData->buf) = sqrt(sumSquares) + 100; + resultData->bufLen = sizeof(double); + resultData->numOfResult = 1; + return 0; +} From e2a20c1d159354f00b2088b2d0529ef8a53215e6 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 20 Apr 2023 14:38:23 +0800 Subject: [PATCH 010/109] test: add udf2_dup test case --- tests/system-test/0-others/udf_create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/0-others/udf_create.py b/tests/system-test/0-others/udf_create.py index ec7716fba3..ee1a0ef5b3 100644 --- a/tests/system-test/0-others/udf_create.py +++ b/tests/system-test/0-others/udf_create.py @@ -257,7 +257,7 @@ class TDTestCase: # aggregate functions - tdSql.query("select udf2(num1) ,udf2_dup(num2) from tb") + tdSql.query("select udf2(num1) ,udf2_dup(num1) from tb") val = tdSql.queryResult[0][0] + 100 tdSql.checkData(0,1,val) From 3a9dddd6199d9faa503365283cdc26cd7c655524 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 20 Apr 2023 15:11:12 +0800 Subject: [PATCH 011/109] enhance: modify inside isEligibleAgg func --- source/libs/planner/src/planOptimizer.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index e100a67b93..effbbc161e 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2526,13 +2526,11 @@ static bool tbCntScanOptIsEligibleAggFuncs(SNodeList* pAggFuncs) { return false; } } - return true; + return LIST_LENGTH(pAggFuncs) > 0; } static bool tbCntScanOptIsEligibleAgg(SAggLogicNode* pAgg) { - return tbCntScanOptIsEligibleGroupKeys(pAgg->pGroupKeys) && - (NULL != pAgg->pAggFuncs) && - tbCntScanOptIsEligibleAggFuncs(pAgg->pAggFuncs); + return tbCntScanOptIsEligibleGroupKeys(pAgg->pGroupKeys) && tbCntScanOptIsEligibleAggFuncs(pAgg->pAggFuncs); } static bool tbCntScanOptGetColValFromCond(SOperatorNode* pOper, SColumnNode** pCol, SValueNode** pVal) { From b1561c95c33783be8478eb5b0fe1a8ddb0d5fe4e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 20 Apr 2023 15:31:18 +0800 Subject: [PATCH 012/109] fix:add test cases for escape in schemaless --- source/client/src/clientSml.c | 2 + source/client/src/clientSmlLine.c | 2 +- source/client/src/clientTmq.c | 3 - source/client/test/smlTest.cpp | 36 ++++++++-- source/common/src/tglobal.c | 2 +- utils/test/c/sml_test.c | 107 ++++++++++++++++++++++++++++++ 6 files changed, 140 insertions(+), 12 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 855e48b88c..c972575e59 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1175,6 +1175,7 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) { } for (size_t i = 0; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); + terrno = 0; taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); if (terrno == TSDB_CODE_DUP_KEY) { return terrno; @@ -1250,6 +1251,7 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat); + terrno = 0; smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags); if (terrno == TSDB_CODE_DUP_KEY) { return terrno; diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index f4f4a5b630..da1bba19dc 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -465,7 +465,7 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin value = tmp; } - SSmlKv kv = {.key = key, .keyLen = keyLen, .value = value, .length = valueLen}; + SSmlKv kv = {.key = key, .keyLen = keyLen, .value = value, .length = valueLen, .keyEscaped = keyEscaped, .valueEscaped = valueEscaped}; int32_t ret = smlParseValue(&kv, &info->msgBuf); if (ret != TSDB_CODE_SUCCESS) { smlBuildInvalidDataMsg(&info->msgBuf, "smlParseValue error", value); diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 6642e716e6..16a4f55840 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -373,9 +373,6 @@ int32_t tmq_list_append(tmq_list_t* list, const char* src) { SArray* container = &list->container; if (src == NULL || src[0] == 0) return -1; char* topic = taosStrdup(src); - if (topic[0] != '`') { - strtolower(topic, src); - } if (taosArrayPush(container, &topic) == NULL) return -1; return 0; } diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 76911e229a..1578b8b607 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -50,8 +50,9 @@ TEST(testCase, smlParseInfluxString_Test) { int ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql); - ASSERT_EQ(elements.measureLen, strlen(",st")); - ASSERT_EQ(elements.measureTagsLen, strlen(",st,t1=3,t2=4,t3=t3")); + ASSERT_EQ(elements.measureLen, strlen("\\,st")); + ASSERT_EQ(elements.measureEscaped, true); + ASSERT_EQ(elements.measureTagsLen, strlen("\\,st,t1=3,t2=4,t3=t3")); ASSERT_EQ(elements.tags, sql + elements.measureLen + 1); ASSERT_EQ(elements.tagsLen, strlen("t1=3,t2=4,t3=t3")); @@ -204,7 +205,26 @@ TEST(testCase, smlParseCols_Error_Test) { "st,t=1 c=-3.402823466e+39u64 1626006833639000000", "st,t=1 c=-339u64 1626006833639000000", "st,t=1 c=18446744073709551616u64 1626006833639000000", - "st,t=1 c=1=2 1626006833639000000"}; + "st,t=1 c=1=2 1626006833639000000,", + // escape error test + // measure comma,space + "s,t,t=1 c=1 1626006833639000000,", + "s t,t=1 c=1 1626006833639000000,", + //tag key comma,equal,space + "st,t,t=1 c=2 1626006833639000000,", + "st,t=t=1 c=2 1626006833639000000,", + "st,t t=1 c=2 1626006833639000000,", + //tag value comma,equal,space + "st,tt=a,a c=2 1626006833639000000,", + "st,t=t=a a c=2 1626006833639000000,", + "st,t t=a=a c=2 1626006833639000000,", + //field key comma,equal,space + "st,tt=aa c,1=2 1626006833639000000,", + "st,tt=aa c=1=2 1626006833639000000,", + "st,tt=aa c 1=2 1626006833639000000,", + //field value double quote,slash + "st,tt=aa c=\"a\"a\" 1626006833639000000,", + }; SSmlHandle *info = smlBuildSmlInfo(NULL); info->protocol = TSDB_SML_LINE_PROTOCOL; @@ -256,16 +276,18 @@ TEST(testCase, smlParseCols_Test) { ASSERT_EQ(strncasecmp(kv->key, "cb=in", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BINARY); - ASSERT_EQ(kv->length, 17); - ASSERT_EQ(strncasecmp(kv->value, "pass,it ", 8), 0); + ASSERT_EQ(kv->length, 18); + ASSERT_EQ(kv->keyEscaped, true); + ASSERT_EQ(kv->valueEscaped, false); + ASSERT_EQ(strncasecmp(kv->value, "pass\\,it ", 9), 0); // nchar kv = (SSmlKv *)taosArrayGet(elements.colArray, 2); ASSERT_EQ(strncasecmp(kv->key, "cnch", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); - ASSERT_EQ(kv->length, 8); - ASSERT_EQ(strncasecmp(kv->value, "ii=sd", 5), 0); + ASSERT_EQ(kv->length, 9); + ASSERT_EQ(strncasecmp(kv->value, "ii\\=sd", 5), 0); // bool kv = (SSmlKv *)taosArrayGet(elements.colArray, 3); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index da4a912238..aa35b298e6 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -198,7 +198,7 @@ int32_t tsTransPullupInterval = 2; int32_t tsMqRebalanceInterval = 2; int32_t tsStreamCheckpointTickInterval = 1; int32_t tsTtlUnit = 86400; -int32_t tsTtlPushInterval = 86400; +int32_t tsTtlPushInterval = 3600; int32_t tsGrantHBInterval = 60; int32_t tsUptimeInterval = 300; // seconds char tsUdfdResFuncs[512] = ""; // udfd resident funcs that teardown when udfd exits diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 755ab55625..515da64081 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -895,6 +895,61 @@ int smlProcess_18784_Test() { return code; } +int sml_escape_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists db_escape"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use db_escape"); + taos_free_result(pRes); + + const char *sql[] = { + "d\\,i=\\ s\\k\",dev\"i\\,\\=\\ ce=s\"i\\,\\=\\ dc inode\"i\\,\\=\\ s_used=176059i,total=1076048383523889174i 1661943960000000000", + "d\\,i=\\ s\\k\",dev\"i\\,\\=\\ ce=s\"i\\,\\=\\ dc inode\"i\\,\\=\\ s_free=\"\\\"id,= ei\\\\\\f\" 1661943960000000000", + }; + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, 0); + printf("%s result:%s, rows:%d\n", __FUNCTION__, taos_errstr(pRes), taos_affected_rows(pRes)); + int code = taos_errno(pRes); + ASSERT(!code); + ASSERT(taos_affected_rows(pRes) == 1); + taos_free_result(pRes); + + pRes = taos_query(taos, "select * from `d,i= s\\k\"`"); //check stable name + ASSERT(pRes); + int fieldNum = taos_field_count(pRes); + ASSERT(fieldNum == 5); + printf("fieldNum:%d\n", fieldNum); + + int numFields = taos_num_fields(pRes); + TAOS_FIELD *fields = taos_fetch_fields(pRes); + ASSERT(numFields == 5); + ASSERT(strcmp(fields[4].name, "dev\"i,= ce") == 0); + + TAOS_ROW row = NULL; + int32_t rowIndex = 0; + while ((row = taos_fetch_row(pRes)) != NULL) { + int64_t ts = *(int64_t *)row[0]; + int64_t used = *(int64_t *)row[1]; + int64_t total = *(int64_t *)row[2]; + int64_t freed = *(int64_t *)row[3]; + + if (rowIndex == 0) { + ASSERT(ts == 1661943960000); + ASSERT(used == 176059); + ASSERT(total == 1076048383523889174); + ASSERT(freed == 66932805); + ASSERT(strcmp(row[4], "s\"i,= dc") == 0); + + } + rowIndex++; + } + taos_free_result(pRes); + taos_close(taos); + + return code; +} + int sml_19221_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -961,6 +1016,55 @@ int sml_ts2164_Test() { return code; } + +int sml_ts3116_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = + taos_query(taos, "DROP DATABASE IF EXISTS ts3116"); + taos_free_result(pRes); + + pRes = taos_query(taos, "CREATE DATABASE IF NOT EXISTS ts3116 BUFFER 384 MINROWS 1000 PAGES 256 PRECISION 'ns'"); + taos_free_result(pRes); + + char *sql = { + "meters,location=la,groupid=ca current=11.8,voltage=221", + }; + + pRes = taos_query(taos, "use ts3116"); + taos_free_result(pRes); + int32_t totalRows = 0; + char *tmp = (char *)taosMemoryCalloc(1024, 1); + memcpy(tmp, sql, strlen(sql)); + totalRows = 0; + pRes = taos_schemaless_insert_raw(taos, tmp, strlen(tmp), &totalRows, TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + taosMemoryFree(tmp); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + + char *sql1 = { + "meters,location=la,groupid=ca\\=3 current=11.8,voltage=221\nmeters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27", + }; + + pRes = taos_query(taos, "use ts3116"); + taos_free_result(pRes); + + tmp = (char *)taosMemoryCalloc(1024, 1); + memcpy(tmp, sql1, strlen(sql1)); + totalRows = 0; + pRes = taos_schemaless_insert_raw(taos, tmp, strlen(tmp), &totalRows, TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + taosMemoryFree(tmp); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + code = taos_errno(pRes); + taos_free_result(pRes); + taos_close(taos); + + return code; +} + int sml_td22898_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -1195,6 +1299,9 @@ int main(int argc, char *argv[]) { } int ret = 0; + ret = sml_escape_Test(); + ret = sml_ts3116_Test(); + ASSERT(!ret); ret = sml_ts2385_Test(); // this test case need config sml table name using ./sml_test config_file ASSERT(!ret); // for(int i = 0; i < sizeof(str)/sizeof(str[0]); i++){ From b60b1796f7a6b08a606c57e6203ebe20f607b213 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 20 Apr 2023 15:53:41 +0800 Subject: [PATCH 013/109] fix(stream): add lock during check wal to create new stream task. --- source/dnode/vnode/src/tq/tqRestore.c | 82 +++++++++++++++------------ source/libs/stream/src/streamMeta.c | 6 -- 2 files changed, 45 insertions(+), 43 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqRestore.c b/source/dnode/vnode/src/tq/tqRestore.c index 3a4bb65c0a..657dd376a1 100644 --- a/source/dnode/vnode/src/tq/tqRestore.c +++ b/source/dnode/vnode/src/tq/tqRestore.c @@ -15,8 +15,7 @@ #include "tq.h" -static int32_t streamTaskReplayWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetStore, bool* pScanIdle); -static int32_t transferToNormalTask(SStreamMeta* pStreamMeta, SArray* pTaskList); +static int32_t doCreateReqsByScanWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetStore, bool* pScanIdle); // this function should be executed by stream threads. // there is a case that the WAL increases more fast than the restore procedure, and this restore procedure @@ -32,7 +31,7 @@ int tqStreamTasksScanWal(STQ* pTq) { // check all restore tasks bool shouldIdle = true; - streamTaskReplayWal(pTq->pStreamMeta, pTq->pOffsetStore, &shouldIdle); + doCreateReqsByScanWal(pTq->pStreamMeta, pTq->pOffsetStore, &shouldIdle); int32_t times = 0; @@ -55,50 +54,50 @@ int tqStreamTasksScanWal(STQ* pTq) { int64_t el = (taosGetTimestampMs() - st); tqDebug("vgId:%d scan wal for stream tasks completed, elapsed time:%"PRId64" ms", vgId, el); - - // restore wal scan flag -// atomic_store_8(&pTq->pStreamMeta->walScan, 0); return 0; } -//int32_t transferToNormalTask(SStreamMeta* pStreamMeta, SArray* pTaskList) { -// int32_t numOfTask = taosArrayGetSize(pTaskList); -// if (numOfTask <= 0) { -// return TSDB_CODE_SUCCESS; -// } -// -// // todo: add lock -// for (int32_t i = 0; i < numOfTask; ++i) { -// SStreamTask* pTask = taosArrayGetP(pTaskList, i); -// tqDebug("vgId:%d transfer s-task:%s state restore -> ready, checkpoint:%" PRId64 " checkpoint id:%" PRId64, -// pStreamMeta->vgId, pTask->id.idStr, pTask->chkInfo.version, pTask->chkInfo.id); -// taosHashRemove(pStreamMeta->pWalReadTasks, &pTask->id.taskId, sizeof(pTask->id.taskId)); -// -// // NOTE: do not change the following order -// atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__NORMAL); -// taosHashPut(pStreamMeta->pTasks, &pTask->id.taskId, sizeof(pTask->id.taskId), &pTask, POINTER_BYTES); -// } -// -// return TSDB_CODE_SUCCESS; -//} - -int32_t streamTaskReplayWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetStore, bool* pScanIdle) { +static SArray* extractTaskIdList(SStreamMeta* pStreamMeta, int32_t numOfTasks) { + SArray* pTaskIdList = taosArrayInit(numOfTasks, sizeof(int32_t)); void* pIter = NULL; - int32_t vgId = pStreamMeta->vgId; - *pScanIdle = true; - - bool allWalChecked = true; - tqDebug("vgId:%d start to check wal to extract new submit block", vgId); - - while (1) { + taosWLockLatch(&pStreamMeta->lock); + while(1) { pIter = taosHashIterate(pStreamMeta->pTasks, pIter); if (pIter == NULL) { break; } SStreamTask* pTask = *(SStreamTask**)pIter; + taosArrayPush(pTaskIdList, &pTask->id.taskId); + } + + taosWUnLockLatch(&pStreamMeta->lock); + return pTaskIdList; +} + +int32_t doCreateReqsByScanWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetStore, bool* pScanIdle) { + *pScanIdle = true; + bool noNewDataInWal = true; + int32_t vgId = pStreamMeta->vgId; + + int32_t numOfTasks = taosHashGetSize(pStreamMeta->pTasks); + if (numOfTasks == 0) { + return TSDB_CODE_SUCCESS; + } + + tqDebug("vgId:%d start to check wal to extract new submit block for %d tasks", vgId, numOfTasks); + SArray* pTaskIdList = extractTaskIdList(pStreamMeta, numOfTasks); + + for (int32_t i = 0; i < numOfTasks; ++i) { + int32_t* pTaskId = taosArrayGet(pTaskIdList, i); + SStreamTask* pTask = streamMetaAcquireTask(pStreamMeta, *pTaskId); + if (pTask == NULL) { + continue; + } + if (pTask->taskLevel != TASK_LEVEL__SOURCE) { + streamMetaReleaseTask(pStreamMeta, pTask); continue; } @@ -106,6 +105,7 @@ int32_t streamTaskReplayWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetSto pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) { tqDebug("s-task:%s skip push data, not ready for processing, status %d", pTask->id.idStr, pTask->status.taskStatus); + streamMetaReleaseTask(pStreamMeta, pTask); continue; } @@ -115,6 +115,7 @@ int32_t streamTaskReplayWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetSto if (tInputQueueIsFull(pTask)) { tqDebug("vgId:%d s-task:%s input queue is full, do nothing", vgId, pTask->id.idStr); + streamMetaReleaseTask(pStreamMeta, pTask); continue; } @@ -127,6 +128,7 @@ int32_t streamTaskReplayWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetSto // seek the stored version and extract data from WAL int32_t code = walReadSeekVer(pTask->exec.pWalReader, pOffset->val.version); if (code != TSDB_CODE_SUCCESS) { // no data in wal, quit + streamMetaReleaseTask(pStreamMeta, pTask); continue; } @@ -136,6 +138,7 @@ int32_t streamTaskReplayWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetSto SPackedData packData = {0}; code = extractSubmitMsgFromWal(pTask->exec.pWalReader, &packData); if (code != TSDB_CODE_SUCCESS) { // failed, continue + streamMetaReleaseTask(pStreamMeta, pTask); continue; } @@ -143,10 +146,11 @@ int32_t streamTaskReplayWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetSto if (p == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tqError("%s failed to create data submit for stream since out of memory", pTask->id.idStr); + streamMetaReleaseTask(pStreamMeta, pTask); continue; } - allWalChecked = false; + noNewDataInWal = false; tqDebug("s-task:%s submit data extracted from WAL", pTask->id.idStr); code = tqAddInputBlockNLaunchTask(pTask, (SStreamQueueItem*)p, packData.ver); @@ -160,11 +164,15 @@ int32_t streamTaskReplayWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetSto streamDataSubmitDestroy(p); taosFreeQitem(p); + streamMetaReleaseTask(pStreamMeta, pTask); } - if (allWalChecked) { + // all wal are checked, and no new data available in wal. + if (noNewDataInWal) { *pScanIdle = true; } + + taosArrayDestroy(pTaskIdList); return 0; } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index c9ea0c382a..90c06dbd69 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -84,11 +84,6 @@ void streamMetaClose(SStreamMeta* pMeta) { tdbClose(pMeta->db); void* pIter = NULL; -// while(pMeta->walScan) { -// qDebug("wait stream daemon quit"); -// taosMsleep(100); -// } - while (1) { pIter = taosHashIterate(pMeta->pTasks, pIter); if (pIter == NULL) { @@ -102,7 +97,6 @@ void streamMetaClose(SStreamMeta* pMeta) { } tFreeStreamTask(pTask); - /*streamMetaReleaseTask(pMeta, pTask);*/ } taosHashCleanup(pMeta->pTasks); From 8e8a3b268b7509f42a3d065899ceb7448a9301c8 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 20 Apr 2023 15:54:19 +0800 Subject: [PATCH 014/109] fix:add test cases for escape in schemaless --- source/client/src/clientSmlLine.c | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index da1bba19dc..6260bcb234 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -452,25 +452,27 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin return TSDB_CODE_SML_INVALID_DATA; } + SSmlKv kv = {.key = key, .keyLen = keyLen, .value = value, .length = valueLen}; + int32_t ret = smlParseValue(&kv, &info->msgBuf); + if (ret != TSDB_CODE_SUCCESS) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlParseValue error", value); + return ret; + } + if (keyEscaped){ char *tmp = (char*)taosMemoryMalloc(keyLen); memcpy(tmp, key, keyLen); PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, keyLen); - key = tmp; - } - if (valueEscaped){ - char *tmp = (char*)taosMemoryMalloc(valueLen); - memcpy(tmp, value, valueLen); - PROCESS_SLASH_IN_FIELD_VALUE(tmp, valueLen); - value = tmp; + kv.key = tmp; + kv.keyEscaped = keyEscaped; } - SSmlKv kv = {.key = key, .keyLen = keyLen, .value = value, .length = valueLen, .keyEscaped = keyEscaped, .valueEscaped = valueEscaped}; - int32_t ret = smlParseValue(&kv, &info->msgBuf); - if (ret != TSDB_CODE_SUCCESS) { - smlBuildInvalidDataMsg(&info->msgBuf, "smlParseValue error", value); - freeSSmlKv(&kv); - return ret; + if (valueEscaped){ + char *tmp = (char*)taosMemoryMalloc(kv.length); + memcpy(tmp, kv.value, kv.length); + PROCESS_SLASH_IN_FIELD_VALUE(tmp, kv.length); + kv.value = tmp; + kv.valueEscaped = valueEscaped; } if (info->dataFormat) { From 1b2fe38f75e51faa9a748b8f25975565245527d7 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 20 Apr 2023 16:22:16 +0800 Subject: [PATCH 015/109] enh: improve logging msg in metaSnapWrite --- include/util/tutil.h | 2 ++ source/dnode/vnode/src/meta/metaSnapshot.c | 7 +++-- source/dnode/vnode/src/meta/metaTable.c | 35 +++++++++++++++------- 3 files changed, 31 insertions(+), 13 deletions(-) diff --git a/include/util/tutil.h b/include/util/tutil.h index e0801e5295..c3eff5cde3 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -98,6 +98,8 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, goto LABEL; \ } +#define VND_CHECK_CODE(CODE, LINO, LABEL) TSDB_CHECK_CODE(CODE, LINO, LABEL) + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c index 0126d29cc9..56e802d4fb 100644 --- a/source/dnode/vnode/src/meta/metaSnapshot.c +++ b/source/dnode/vnode/src/meta/metaSnapshot.c @@ -187,23 +187,24 @@ _err: int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { int32_t code = 0; + int32_t line = 0; SMeta* pMeta = pWriter->pMeta; SMetaEntry metaEntry = {0}; SDecoder* pDecoder = &(SDecoder){0}; tDecoderInit(pDecoder, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)); code = metaDecodeEntry(pDecoder, &metaEntry); - if (code) goto _err; + VND_CHECK_CODE(code, line, _err); code = metaHandleEntry(pMeta, &metaEntry); - if (code) goto _err; + VND_CHECK_CODE(code, line, _err); tDecoderClear(pDecoder); return code; _err: tDecoderClear(pDecoder); - metaError("vgId:%d, vnode snapshot meta write failed since %s", TD_VID(pMeta->pVnode), tstrerror(code)); + metaError("vgId:%d, vnode snapshot meta write failed since %s at line:%d", TD_VID(pMeta->pVnode), terrstr(), line); return code; } diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 3325f4055c..8a311b9b80 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -2065,40 +2065,52 @@ _exit: } int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME) { + int32_t code = 0; + int32_t line = 0; metaWLock(pMeta); // save to table.db - if (metaSaveToTbDb(pMeta, pME) < 0) goto _err; + code = metaSaveToTbDb(pMeta, pME); + VND_CHECK_CODE(code, line, _err); // update uid.idx - if (metaUpdateUidIdx(pMeta, pME) < 0) goto _err; + code = metaUpdateUidIdx(pMeta, pME); + VND_CHECK_CODE(code, line, _err); // update name.idx - if (metaUpdateNameIdx(pMeta, pME) < 0) goto _err; + code = metaUpdateNameIdx(pMeta, pME); + VND_CHECK_CODE(code, line, _err); if (pME->type == TSDB_CHILD_TABLE) { // update ctb.idx - if (metaUpdateCtbIdx(pMeta, pME) < 0) goto _err; + code = metaUpdateCtbIdx(pMeta, pME); + VND_CHECK_CODE(code, line, _err); // update tag.idx - if (metaUpdateTagIdx(pMeta, pME) < 0) goto _err; + code = metaUpdateTagIdx(pMeta, pME); + VND_CHECK_CODE(code, line, _err); } else { // update schema.db - if (metaSaveToSkmDb(pMeta, pME) < 0) goto _err; + code = metaSaveToSkmDb(pMeta, pME); + VND_CHECK_CODE(code, line, _err); if (pME->type == TSDB_SUPER_TABLE) { - if (metaUpdateSuidIdx(pMeta, pME) < 0) goto _err; + code = metaUpdateSuidIdx(pMeta, pME); + VND_CHECK_CODE(code, line, _err); } } - if (metaUpdateCtimeIdx(pMeta, pME) < 0) goto _err; + code = metaUpdateCtimeIdx(pMeta, pME); + VND_CHECK_CODE(code, line, _err); if (pME->type == TSDB_NORMAL_TABLE) { - if (metaUpdateNcolIdx(pMeta, pME) < 0) goto _err; + code = metaUpdateNcolIdx(pMeta, pME); + VND_CHECK_CODE(code, line, _err); } if (pME->type != TSDB_SUPER_TABLE) { - if (metaUpdateTtlIdx(pMeta, pME) < 0) goto _err; + code = metaUpdateTtlIdx(pMeta, pME); + VND_CHECK_CODE(code, line, _err); } metaULock(pMeta); @@ -2106,8 +2118,11 @@ int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME) { _err: metaULock(pMeta); + metaError("vgId:%d, failed to handle meta entry since %s at line:%d, ver:%" PRId64 ", uid:%" PRId64, + TD_VID(pMeta->pVnode), terrstr(), line, pME->version, pME->uid); return -1; } + // refactor later void *metaGetIdx(SMeta *pMeta) { return pMeta->pTagIdx; } void *metaGetIvtIdx(SMeta *pMeta) { return pMeta->pTagIvtIdx; } From 2714da26e7873d3209832fb78adae8c29a80b7f7 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 20 Apr 2023 17:26:54 +0800 Subject: [PATCH 016/109] fix:[TS-3221] reset max stmt if execute error --- source/client/src/clientSml.c | 8 +++++++- source/client/src/clientSmlLine.c | 12 ++++++------ utils/test/c/sml_test.c | 9 ++++++--- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index c972575e59..065f2e1fdb 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1316,9 +1316,15 @@ static int32_t smlInsertData(SSmlHandle *info) { uDebug("SML:0x%" PRIx64 " smlInsertData table:%s, uid:%" PRIu64 ", format:%d", info->id, pName.tname, tableData->uid, info->dataFormat); + int measureLen = tableData->sTableNameLen; + char* measure = (char*)taosMemoryMalloc(tableData->sTableNameLen); + memcpy(measure, tableData->sTableName, tableData->sTableNameLen); + PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); + code = smlBindData(info->pQuery, info->dataFormat, tableData->tags, (*pMeta)->cols, tableData->cols, - (*pMeta)->tableMeta, tableData->childTableName, tableData->sTableName, tableData->sTableNameLen, + (*pMeta)->tableMeta, tableData->childTableName, measure, measureLen, info->ttl, info->msgBuf.buf, info->msgBuf.len); + taosMemoryFree(measure); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlBindData failed", info->id); return code; diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index 6260bcb234..1732473c11 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -155,7 +155,7 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin char* measure = currElement->measure; int measureLen = currElement->measureLen; if(currElement->measureEscaped){ - measure = taosMemoryMalloc(currElement->measureLen); + measure = (char*)taosMemoryMalloc(currElement->measureLen); memcpy(measure, currElement->measure, currElement->measureLen); PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); } @@ -230,7 +230,7 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin return TSDB_CODE_SML_INVALID_DATA; } - if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(value)) { + if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) { valueLenEscaped++; valueEscaped = true; } @@ -349,7 +349,7 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin char* measure = currElement->measure; int measureLen = currElement->measureLen; if(currElement->measureEscaped){ - measure = taosMemoryMalloc(currElement->measureLen); + measure = (char*)taosMemoryMalloc(currElement->measureLen); memcpy(measure, currElement->measure, currElement->measureLen); PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); } @@ -460,9 +460,9 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin } if (keyEscaped){ - char *tmp = (char*)taosMemoryMalloc(keyLen); - memcpy(tmp, key, keyLen); - PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, keyLen); + char *tmp = (char*)taosMemoryMalloc(kv.keyLen); + memcpy(tmp, key, kv.keyLen); + PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, kv.keyLen); kv.key = tmp; kv.keyEscaped = keyEscaped; } diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 515da64081..54fa594e39 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -906,7 +906,7 @@ int sml_escape_Test() { const char *sql[] = { "d\\,i=\\ s\\k\",dev\"i\\,\\=\\ ce=s\"i\\,\\=\\ dc inode\"i\\,\\=\\ s_used=176059i,total=1076048383523889174i 1661943960000000000", - "d\\,i=\\ s\\k\",dev\"i\\,\\=\\ ce=s\"i\\,\\=\\ dc inode\"i\\,\\=\\ s_free=\"\\\"id,= ei\\\\\\f\" 1661943960000000000", + "d\\,i=\\ s\\k\",dev\"i\\,\\=\\ ce=s\"i\\,\\=\\ dc inode\"i\\,\\=\\ s_f\\\\ree=\"\\\"id,= ei\\\\\\f\" 1661943960000000000", }; pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, 0); printf("%s result:%s, rows:%d\n", __FUNCTION__, taos_errstr(pRes), taos_affected_rows(pRes)); @@ -924,6 +924,9 @@ int sml_escape_Test() { int numFields = taos_num_fields(pRes); TAOS_FIELD *fields = taos_fetch_fields(pRes); ASSERT(numFields == 5); + ASSERT(strcmp(fields[1].name, "inode\"i,= s_used") == 0); + ASSERT(strcmp(fields[2].name, "total") == 0); + ASSERT(strcmp(fields[3].name, "inode\"i,= s_f\\\\ree") == 0); ASSERT(strcmp(fields[4].name, "dev\"i,= ce") == 0); TAOS_ROW row = NULL; @@ -932,13 +935,12 @@ int sml_escape_Test() { int64_t ts = *(int64_t *)row[0]; int64_t used = *(int64_t *)row[1]; int64_t total = *(int64_t *)row[2]; - int64_t freed = *(int64_t *)row[3]; if (rowIndex == 0) { ASSERT(ts == 1661943960000); ASSERT(used == 176059); ASSERT(total == 1076048383523889174); - ASSERT(freed == 66932805); + ASSERT(strcmp(row[3], "\"id,= ei\\\\f") == 0); ASSERT(strcmp(row[4], "s\"i,= dc") == 0); } @@ -1300,6 +1302,7 @@ int main(int argc, char *argv[]) { int ret = 0; ret = sml_escape_Test(); + ASSERT(!ret); ret = sml_ts3116_Test(); ASSERT(!ret); ret = sml_ts2385_Test(); // this test case need config sml table name using ./sml_test config_file From 8323ad86701cc80f0e00251eb9a2a7e36a264bdf Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 20 Apr 2023 17:29:08 +0800 Subject: [PATCH 017/109] enh(stream): add more check to stop stream asap. --- source/dnode/vnode/src/tq/tqRestore.c | 2 +- source/libs/stream/src/streamExec.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqRestore.c b/source/dnode/vnode/src/tq/tqRestore.c index 657dd376a1..9a9c750194 100644 --- a/source/dnode/vnode/src/tq/tqRestore.c +++ b/source/dnode/vnode/src/tq/tqRestore.c @@ -96,7 +96,7 @@ int32_t doCreateReqsByScanWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetS continue; } - if (pTask->taskLevel != TASK_LEVEL__SOURCE) { + if ((pTask->taskLevel != TASK_LEVEL__SOURCE) || (pTask->status.taskStatus == TASK_STATUS__DROPPING)) { streamMetaReleaseTask(pStreamMeta, pTask); continue; } diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 9a6ff302ef..f52af66387 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -368,7 +368,7 @@ int32_t streamTryExec(SStreamTask* pTask) { atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); qDebug("s-task:%s exec completed", pTask->id.idStr); - if (!taosQueueEmpty(pTask->inputQueue->queue)) { + if (!taosQueueEmpty(pTask->inputQueue->queue) && (pTask->status.taskStatus != TASK_STATUS__DROPPING)) { streamSchedExec(pTask); } } From c7e42d5422e1fbc1d732ec3edb180a4713854eb7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 20 Apr 2023 17:30:02 +0800 Subject: [PATCH 018/109] other: do some internal refactor. --- source/dnode/vnode/src/tq/tqRestore.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqRestore.c b/source/dnode/vnode/src/tq/tqRestore.c index 9a9c750194..56f0a80b9e 100644 --- a/source/dnode/vnode/src/tq/tqRestore.c +++ b/source/dnode/vnode/src/tq/tqRestore.c @@ -96,15 +96,14 @@ int32_t doCreateReqsByScanWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetS continue; } - if ((pTask->taskLevel != TASK_LEVEL__SOURCE) || (pTask->status.taskStatus == TASK_STATUS__DROPPING)) { + int32_t status = pTask->status.taskStatus; + if ((pTask->taskLevel != TASK_LEVEL__SOURCE) || (status == TASK_STATUS__DROPPING)) { streamMetaReleaseTask(pStreamMeta, pTask); continue; } - if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE || - pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) { - tqDebug("s-task:%s skip push data, not ready for processing, status %d", pTask->id.idStr, - pTask->status.taskStatus); + if (status == TASK_STATUS__RECOVER_PREPARE || status == TASK_STATUS__WAIT_DOWNSTREAM) { + tqDebug("s-task:%s skip push data, not ready for processing, status %d", pTask->id.idStr, status); streamMetaReleaseTask(pStreamMeta, pTask); continue; } From e36bf05f98b7b973f742df9ab7238a4cad518f3f Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 20 Apr 2023 17:30:09 +0800 Subject: [PATCH 019/109] enh: support get subtable tag and auth --- include/libs/catalog/catalog.h | 1 + source/libs/catalog/inc/catalogInt.h | 10 ++ source/libs/catalog/src/catalog.c | 2 +- source/libs/catalog/src/ctgAsync.c | 96 +++++++++++++++++--- source/libs/catalog/src/ctgCache.c | 23 +++++ source/libs/catalog/src/ctgUtil.c | 131 ++++++++++++++++++--------- 6 files changed, 202 insertions(+), 61 deletions(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 2f8e7846f3..6f2fb4eb6b 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -106,6 +106,7 @@ typedef struct SMetaData { SArray* pUser; // pRes = SUserAuthRes* SArray* pQnodeList; // pRes = SArray* SArray* pTableCfg; // pRes = STableCfg* + SArray* pTableTag; // pRes = SArray* SArray* pDnodeList; // pRes = SArray* SMetaRes* pSvrVer; // pRes = char* } SMetaData; diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index f0e5024c59..1eaf45dafe 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -154,6 +154,11 @@ typedef struct SCtgTbCacheInfo { int32_t tbType; } SCtgTbCacheInfo; +typedef struct SCtgTbMetaParam { + SName* pName; + int32_t flag; +} SCtgTbMetaParam; + typedef struct SCtgTbMetaCtx { SCtgTbCacheInfo tbInfo; int32_t vgId; @@ -631,6 +636,7 @@ typedef struct SCtgCacheItemInfo { #define CTG_FLAG_SYS_DB 0x8 #define CTG_FLAG_FORCE_UPDATE 0x10 #define CTG_FLAG_ONLY_CACHE 0x20 +#define CTG_FLAG_SYNC_OP 0x40 #define CTG_FLAG_SET(_flag, _v) ((_flag) |= (_v)) @@ -933,6 +939,10 @@ void ctgReleaseVgMetaToCache(SCatalog* pCtg, SCtgDBCache* dbCache, SCtgTbCach void ctgReleaseTbMetaToCache(SCatalog* pCtg, SCtgDBCache* dbCache, SCtgTbCache* pCache); void ctgGetGlobalCacheStat(SCtgCacheStat* pStat); int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res); +int32_t ctgGetTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta); +int32_t ctgGetCachedStbNameFromSuid(SCatalog* pCtg, char* dbFName, uint64_t suid, char **stbName); +int32_t ctgGetTbTagCb(SCtgTask* pTask); +int32_t ctgGetUserCb(SCtgTask* pTask); extern SCatalogMgmt gCtgMgmt; extern SCtgDebug gCTGDebug; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index bddc6c01a7..b263654e70 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -208,7 +208,7 @@ int32_t ctgGetTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* ctx } while (true) { - CTG_ERR_JRET(ctgRefreshTbMeta(pCtg, pConn, ctx, &output, false)); + CTG_ERR_JRET(ctgRefreshTbMeta(pCtg, pConn, ctx, &output, ctx->flag & CTG_FLAG_SYNC_OP)); if (CTG_IS_META_TABLE(output->metaType)) { *pTableMeta = output->tbMeta; diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index b10da5cc73..affcfeb8ac 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -21,7 +21,8 @@ #include "trpc.h" int32_t ctgInitGetTbMetaTask(SCtgJob* pJob, int32_t taskIdx, void* param) { - SName* name = (SName*)param; + SCtgTbMetaParam* pParam = (SCtgTbMetaParam*)param; + SName* name = pParam->pName; SCtgTask task = {0}; task.type = CTG_TASK_GET_TB_META; @@ -41,7 +42,7 @@ int32_t ctgInitGetTbMetaTask(SCtgJob* pJob, int32_t taskIdx, void* param) { } memcpy(ctx->pName, name, sizeof(*name)); - ctx->flag = CTG_FLAG_UNKNOWN_STB; + ctx->flag = pParam->flag | CTG_FLAG_UNKNOWN_STB; taosArrayPush(pJob->pTasks, &task); @@ -545,7 +546,7 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const int32_t dbInfoNum = (int32_t)taosArrayGetSize(pReq->pDbInfo); int32_t tbIndexNum = (int32_t)taosArrayGetSize(pReq->pTableIndex); int32_t tbCfgNum = (int32_t)taosArrayGetSize(pReq->pTableCfg); - int32_t tbTagNum = (int32_t)ctgGetTablesReqNum(pReq->pTableTag); + int32_t tbTagNum = (int32_t)taosArrayGetSize(pReq->pTableTag); int32_t taskNum = tbMetaNum + dbVgNum + udfNum + tbHashNum + qnodeNum + dnodeNum + svrVerNum + dbCfgNum + indexNum + userNum + dbInfoNum + tbIndexNum + tbCfgNum + tbTagNum; @@ -646,7 +647,7 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_CFG, name, NULL)); } - for (int32_t i = 0; i < tbCfgNum; ++i) { + for (int32_t i = 0; i < tbTagNum; ++i) { SName* name = taosArrayGet(pReq->pTableTag, i); CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_TAG, name, NULL)); } @@ -761,7 +762,11 @@ int32_t ctgDumpTbHashsRes(SCtgTask* pTask) { int32_t ctgDumpTbIndexRes(SCtgTask* pTask) { SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pTableIndex) { - pJob->jobRes.pTableIndex = taosArrayInit(pJob->tbIndexNum, sizeof(SMetaRes)); + SArray* pRes = taosArrayInit(pJob->tbIndexNum, sizeof(SMetaRes)); + if (atomic_val_compare_exchange_ptr(&pJob->jobRes.pTableIndex, NULL, pRes)) { + taosArrayDestroy(pRes); + } + if (NULL == pJob->jobRes.pTableIndex) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -776,7 +781,11 @@ int32_t ctgDumpTbIndexRes(SCtgTask* pTask) { int32_t ctgDumpTbCfgRes(SCtgTask* pTask) { SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pTableCfg) { - pJob->jobRes.pTableCfg = taosArrayInit(pJob->tbCfgNum, sizeof(SMetaRes)); + SArray* pRes = taosArrayInit(pJob->tbCfgNum, sizeof(SMetaRes)); + if (atomic_val_compare_exchange_ptr(&pJob->jobRes.pTableCfg, NULL, pRes)) { + taosArrayDestroy(pRes); + } + if (NULL == pJob->jobRes.pTableCfg) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -788,6 +797,26 @@ int32_t ctgDumpTbCfgRes(SCtgTask* pTask) { return TSDB_CODE_SUCCESS; } +int32_t ctgDumpTbTagRes(SCtgTask* pTask) { + SCtgJob* pJob = pTask->pJob; + if (NULL == pJob->jobRes.pTableTag) { + SArray* pRes = taosArrayInit(pJob->tbTagNum, sizeof(SMetaRes)); + if (atomic_val_compare_exchange_ptr(&pJob->jobRes.pTableTag, NULL, pRes)) { + taosArrayDestroy(pRes); + } + + if (NULL == pJob->jobRes.pTableTag) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + } + + SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; + taosArrayPush(pJob->jobRes.pTableTag, &res); + + return TSDB_CODE_SUCCESS; +} + + int32_t ctgDumpIndexRes(SCtgTask* pTask) { SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pIndex) { @@ -1123,7 +1152,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; - ctgUpdateTbMetaToCache(pCtg, pOut, false); + ctgUpdateTbMetaToCache(pCtg, pOut, flag & CTG_FLAG_SYNC_OP); if (CTG_IS_META_BOTH(pOut->metaType)) { memcpy(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); @@ -1525,11 +1554,36 @@ _return: int32_t ctgHandleGetTbTagRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; + SCatalog* pCtg = pTask->pJob->pCtg; CTG_ERR_JRET(ctgProcessRspMsg(&pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); STableCfgRsp* pRsp = (STableCfgRsp*)pTask->msgCtx.out; + if (NULL == pRsp->pTags || pRsp->tagsLen <= 0) { + ctgError("invalid tag in tbCfg rsp, pTags:%p, len:%d", pRsp->pTags, pRsp->tagsLen); + CTG_ERR_JRET(TSDB_CODE_INVALID_MSG); + } + + SArray* pTagVals = NULL; + STag* pTag = (STag*)pRsp->pTags; + + if (tTagIsJson(pTag)) { + pTagVals = taosArrayInit(1, sizeof(STagVal)); + if (NULL == pTagVals) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + char* pJson = parseTagDatatoJson(pTag); + STagVal tagVal; + tagVal.cid = 0; + tagVal.type = TSDB_DATA_TYPE_JSON; + tagVal.pData = pJson; + tagVal.nData = strlen(pJson); + taosArrayPush(pTagVals, &tagVal); + } else { + CTG_ERR_JRET(tTagToValArray((const STag*)pRsp->pTags, &pTagVals)); + } - TSWAP(pTask->res, pTask->msgCtx.out); + pTask->res = pTagVals; _return: @@ -1971,7 +2025,10 @@ int32_t ctgLaunchGetTbCfgTask(SCtgTask* pTask) { if (pCtx->tbType <= 0) { CTG_ERR_JRET(ctgReadTbTypeFromCache(pCtg, dbFName, pCtx->pName->tname, &pCtx->tbType)); if (pCtx->tbType <= 0) { - CTG_ERR_JRET(ctgLaunchSubTask(pTask, CTG_TASK_GET_TB_META, ctgGetTbCfgCb, pCtx->pName)); + SCtgTbMetaParam param; + param.pName = pCtx->pName; + param.flag = 0; + CTG_ERR_JRET(ctgLaunchSubTask(pTask, CTG_TASK_GET_TB_META, ctgGetTbCfgCb, ¶m)); return TSDB_CODE_SUCCESS; } } @@ -2019,7 +2076,7 @@ int32_t ctgLaunchGetTbTagTask(SCtgTask* pTask) { if (NULL == pCtx->pVgInfo) { CTG_ERR_JRET(ctgGetTbHashVgroupFromCache(pCtg, pCtx->pName, &pCtx->pVgInfo)); if (NULL == pCtx->pVgInfo) { - CTG_ERR_JRET(ctgLaunchSubTask(pTask, CTG_TASK_GET_DB_VGROUP, ctgGetTbCfgCb, dbFName)); + CTG_ERR_JRET(ctgLaunchSubTask(pTask, CTG_TASK_GET_DB_VGROUP, ctgGetTbTagCb, dbFName)); return TSDB_CODE_SUCCESS; } } @@ -2189,7 +2246,10 @@ int32_t ctgLaunchGetUserTask(SCtgTask* pTask) { taosMemoryFreeClear(rsp.pRawRes); if (rsp.metaNotExists) { - CTG_ERR_RET(ctgLaunchSubTask(pTask, CTG_TASK_GET_TB_META, ctgGetTbCfgCb, &pCtx->user.tbName)); + SCtgTbMetaParam param; + param.pName = &pCtx->user.tbName; + param.flag = CTG_FLAG_SYNC_OP; + CTG_ERR_RET(ctgLaunchSubTask(pTask, CTG_TASK_GET_TB_META, ctgGetUserCb, ¶m)); } else { CTG_ERR_RET(ctgGetUserDbAuthFromMnode(pCtg, pConn, pCtx->user.user, NULL, pTask)); } @@ -2251,9 +2311,11 @@ int32_t ctgGetTbTagCb(SCtgTask* pTask) { SCtgTbTagCtx* pCtx = (SCtgTbTagCtx*)pTask->taskCtx; SDBVgInfo* pDb = (SDBVgInfo*)pTask->subRes.res; - pCtx->pVgInfo = taosMemoryCalloc(1, sizeof(SVgroupInfo)); - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pTask->pJob->pCtg, pDb, pCtx->pName, pCtx->pVgInfo)); - + if (NULL == pCtx->pVgInfo) { + pCtx->pVgInfo = taosMemoryCalloc(1, sizeof(SVgroupInfo)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pTask->pJob->pCtg, pDb, pCtx->pName, pCtx->pVgInfo)); + } + CTG_RET(ctgLaunchGetTbTagTask(pTask)); _return: @@ -2286,8 +2348,12 @@ int32_t ctgCompDbVgTasks(SCtgTask* pTask, void* param, bool* equal) { int32_t ctgCompTbMetaTasks(SCtgTask* pTask, void* param, bool* equal) { SCtgTbMetaCtx* ctx = pTask->taskCtx; + SCtgTbMetaParam* pParam = (SCtgTbMetaParam*)param; - *equal = tNameTbNameEqual(ctx->pName, (SName*)param); + *equal = tNameTbNameEqual(ctx->pName, (SName*)pParam->pName); + if (*equal) { + ctx->flag |= pParam->flag; + } return TSDB_CODE_SUCCESS; } diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 592b6e9c72..ee864d985e 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -703,6 +703,29 @@ _return: CTG_RET(code); } +int32_t ctgGetCachedStbNameFromSuid(SCatalog* pCtg, char* dbFName, uint64_t suid, char **stbName) { + *stbName = NULL; + + SCtgDBCache *dbCache = NULL; + ctgAcquireDBCache(pCtg, dbFName, &dbCache); + if (NULL == dbCache) { + ctgDebug("db %s not in cache", dbFName); + return TSDB_CODE_SUCCESS; + } + + char *stb = taosHashAcquire(dbCache->stbCache, &suid, sizeof(suid)); + if (NULL == stb) { + ctgDebug("stb 0x%" PRIx64 " not in cache, dbFName:%s", suid, dbFName); + return TSDB_CODE_SUCCESS; + } + + *stbName = taosStrdup(stb); + + taosHashRelease(dbCache->stbCache, stb); + + return TSDB_CODE_SUCCESS; +} + int32_t ctgChkAuthFromCache(SCatalog *pCtg, SUserAuthInfo *pReq, bool *inCache, SCtgAuthRsp *pRes) { if (IS_SYS_DBNAME(pReq->tbName.dbname)) { *inCache = true; diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index b2b2b5a87e..7ef2e34d1e 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -486,6 +486,18 @@ void ctgFreeBatchHash(void* hash) { taosMemoryFreeClear(pRes->pRes); } +void ctgFreeJsonTagVal(void *val) { + if (NULL == val) { + return; + } + + STagVal *pVal = (STagVal *)val; + + if (TSDB_DATA_TYPE_JSON == pVal->type) { + taosMemoryFree(pVal->pData); + } +} + void ctgFreeTaskRes(CTG_TASK_TYPE type, void** pRes) { switch (type) { case CTG_TASK_GET_QNODE: @@ -526,6 +538,14 @@ void ctgFreeTaskRes(CTG_TASK_TYPE type, void** pRes) { taosMemoryFreeClear(*pRes); break; } + case CTG_TASK_GET_TB_TAG: { + if (1 == taosArrayGetSize(*pRes)) { + taosArrayDestroyEx(*pRes, ctgFreeJsonTagVal); + } else { + taosArrayDestroy(*pRes); + } + *pRes = NULL; + } case CTG_TASK_GET_TB_META_BATCH: { SArray* pArray = (SArray*)*pRes; int32_t num = taosArrayGetSize(pArray); @@ -679,6 +699,12 @@ void ctgFreeTaskCtx(SCtgTask* pTask) { taosMemoryFreeClear(pTask->taskCtx); break; } + case CTG_TASK_GET_TB_TAG: { + SCtgTbTagCtx* taskCtx = (SCtgTbTagCtx*)pTask->taskCtx; + taosMemoryFreeClear(taskCtx->pName); + taosMemoryFreeClear(taskCtx->pVgInfo); + break; + } case CTG_TASK_GET_DB_VGROUP: case CTG_TASK_GET_DB_CFG: case CTG_TASK_GET_DB_INFO: @@ -1336,54 +1362,69 @@ int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { STableMeta* pMeta = NULL; SGetUserAuthRsp* pInfo = &req->authInfo; SHashObj* pTbs = (AUTH_TYPE_READ == req->singleType) ? pInfo->readTbs : pInfo->writeTbs; + char* stbName = NULL; + + char tbFName[TSDB_TABLE_FNAME_LEN]; + char dbFName[TSDB_DB_FNAME_LEN]; + tNameExtractFullName(&req->pRawReq->tbName, tbFName); + tNameGetFullDbName(&req->pRawReq->tbName, dbFName); - char tbFullName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(&req->pRawReq->tbName, tbFullName); - char* pCond = taosHashGet(pTbs, tbFullName, strlen(tbFullName)); - if (pCond) { - if (strlen(pCond) > 1) { - CTG_ERR_RET(nodesStringToNode(pCond, &res->pRawRes->pCond)); + while (true) { + char* pCond = taosHashGet(pTbs, tbFName, strlen(tbFName)); + if (pCond) { + if (strlen(pCond) > 1) { + CTG_ERR_RET(nodesStringToNode(pCond, &res->pRawRes->pCond)); + } + + res->pRawRes->pass = true; + return TSDB_CODE_SUCCESS; } - res->pRawRes->pass = true; - return TSDB_CODE_SUCCESS; + if (stbName) { + res->pRawRes->pass = false; + goto _return; + } + + CTG_ERR_RET(catalogGetCachedTableMeta(pCtg, &req->pRawReq->tbName, &pMeta)); + if (NULL == pMeta) { + if (req->onlyCache) { + res->metaNotExists = true; + ctgDebug("db %s tb %s meta not in cache for auth", req->pRawReq->tbName.dbname, req->pRawReq->tbName.tname); + return TSDB_CODE_SUCCESS; + } + + SCtgTbMetaCtx ctx = {0}; + ctx.pName = (SName*)&req->pRawReq->tbName; + ctx.flag = CTG_FLAG_UNKNOWN_STB | CTG_FLAG_SYNC_OP; + + CTG_ERR_RET(ctgGetTbMeta(pCtg, req->pConn, &ctx, &pMeta)); + } + + if (TSDB_SUPER_TABLE == pMeta->tableType || TSDB_NORMAL_TABLE == pMeta->tableType) { + res->pRawRes->pass = false; + goto _return; + } + + if (TSDB_CHILD_TABLE == pMeta->tableType) { + CTG_ERR_JRET(ctgGetCachedStbNameFromSuid(pCtg, dbFName, pMeta->suid, &stbName)); + if (NULL == stbName) { + if (req->onlyCache) { + res->metaNotExists = true; + ctgDebug("suid %" PRIu64 " name not in cache for auth", pMeta->suid); + return TSDB_CODE_SUCCESS; + } + + continue; + } + + sprintf(tbFName, "%s.%s", dbFName, stbName); + continue; + } + + ctgError("Invalid table type %d for %s", pMeta->tableType, tbFName); + CTG_ERR_JRET(TSDB_CODE_INVALID_PARA); } - res->pRawRes->pass = false; - - // CTG_ERR_RET(catalogGetCachedTableMeta(pCtg, &req->pRawReq->tbName, &pMeta)); - // if (NULL == pMeta) { - // if (req->onlyCache) { - // res->metaNotExists = true; - // ctgDebug("db %s tb %s meta not in cache for auth", req->pRawReq->tbName.dbname, req->pRawReq->tbName.tname); - // return TSDB_CODE_SUCCESS; - // } - - // CTG_ERR_RET(catalogGetTableMeta(pCtg, req->pConn, &req->pRawReq->tbName, &pMeta)); - // } - - // if (TSDB_SUPER_TABLE == pMeta->tableType || TSDB_NORMAL_TABLE == pMeta->tableType) { - // res->pRawRes->pass = false; - // goto _return; - // } - - // if (TSDB_CHILD_TABLE == pMeta->tableType) { - // res->pRawRes->pass = true; - - // /* - // char stbName[TSDB_TABLE_NAME_LEN] = {0}; - // CTG_ERR_JRET(ctgGetCachedStbNameFromSuid(pCtg, pMeta->suid, stbName)); - // if (0 == stbName[0]) { - // if (req->onlyCache) { - // res->notExists = true; - // return TSDB_CODE_SUCCESS; - // } - - // CTG_ERR_RET(catalogRefreshTableMeta(pCtg, req->pConn, &req->pRawReq->tbName, 0)); - // } - // */ - // } - _return: taosMemoryFree(pMeta); @@ -1423,7 +1464,7 @@ int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { if (pInfo->readTbs && taosHashGetSize(pInfo->readTbs) > 0) { req->singleType = AUTH_TYPE_READ; CTG_ERR_RET(ctgChkSetTbAuthRes(pCtg, req, res)); - if (pRes->pass) { + if (pRes->pass || res->metaNotExists) { return TSDB_CODE_SUCCESS; } } @@ -1439,7 +1480,7 @@ int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { if (pInfo->writeTbs && taosHashGetSize(pInfo->writeTbs) > 0) { req->singleType = AUTH_TYPE_WRITE; CTG_ERR_RET(ctgChkSetTbAuthRes(pCtg, req, res)); - if (pRes->pass) { + if (pRes->pass || res->metaNotExists) { return TSDB_CODE_SUCCESS; } } From dc733352db5164d544084f14b769980adbce8f6b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 20 Apr 2023 17:38:10 +0800 Subject: [PATCH 020/109] enh(stream): set the max input queue size to be 3000. --- source/libs/stream/src/stream.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 0f000f1f50..86ba91f76d 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -16,7 +16,7 @@ #include "streamInc.h" #include "ttimer.h" -#define STREAM_TASK_INPUT_QUEUEU_CAPACITY 100000 +#define STREAM_TASK_INPUT_QUEUEU_CAPACITY 3000 int32_t streamInit() { int8_t old; From ea939c1efb5a769176361e46d0580c114951c2a1 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 20 Apr 2023 17:41:38 +0800 Subject: [PATCH 021/109] fix:add test cases for escape in schemaless --- utils/test/c/sml_test.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 54fa594e39..21964403ea 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -924,10 +924,10 @@ int sml_escape_Test() { int numFields = taos_num_fields(pRes); TAOS_FIELD *fields = taos_fetch_fields(pRes); ASSERT(numFields == 5); - ASSERT(strcmp(fields[1].name, "inode\"i,= s_used") == 0); - ASSERT(strcmp(fields[2].name, "total") == 0); - ASSERT(strcmp(fields[3].name, "inode\"i,= s_f\\\\ree") == 0); - ASSERT(strcmp(fields[4].name, "dev\"i,= ce") == 0); + ASSERT(strncmp(fields[1].name, "inode\"i,= s_used", sizeof("inode\"i,= s_used") - 1) == 0); + ASSERT(strncmp(fields[2].name, "total", sizeof("total") - 1) == 0); + ASSERT(strncmp(fields[3].name, "inode\"i,= s_f\\\\ree", sizeof("inode\"i,= s_f\\\\ree") - 1) == 0); + ASSERT(strncmp(fields[4].name, "dev\"i,= ce", sizeof("dev\"i,= ce") - 1) == 0); TAOS_ROW row = NULL; int32_t rowIndex = 0; @@ -940,8 +940,8 @@ int sml_escape_Test() { ASSERT(ts == 1661943960000); ASSERT(used == 176059); ASSERT(total == 1076048383523889174); - ASSERT(strcmp(row[3], "\"id,= ei\\\\f") == 0); - ASSERT(strcmp(row[4], "s\"i,= dc") == 0); + ASSERT(strncmp(row[3], "\"id,= ei\\\\f", sizeof("\"id,= ei\\\\f") - 1) == 0); + ASSERT(strncmp(row[4], "s\"i,= dc", sizeof("s\"i,= dc") - 1) == 0); } rowIndex++; From 13ae45c3fc6bb5f3dce038569db7f1f40a61032b Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Thu, 20 Apr 2023 17:46:42 +0800 Subject: [PATCH 022/109] test: modify tmq case --- tests/system-test/7-tmq/subscribeDb3.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/system-test/7-tmq/subscribeDb3.py b/tests/system-test/7-tmq/subscribeDb3.py index bddb196f4a..1de9b62bcd 100644 --- a/tests/system-test/7-tmq/subscribeDb3.py +++ b/tests/system-test/7-tmq/subscribeDb3.py @@ -82,7 +82,7 @@ class TDTestCase: tdSql.query("select * from %s.notifyinfo"%cdbName) #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) if tdSql.getRows() == 2 : - print(tdSql.getData(0, 1), tdSql.getData(1, 1)) + tdLog.info("row[0][1]: %d, row[1][1]: %d"%(tdSql.getData(0, 1), tdSql.getData(1, 1))) if tdSql.getData(1, 1) == 1: break time.sleep(0.1) @@ -122,6 +122,7 @@ class TDTestCase: os.system(shellCmd) def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum,rowsPerTbl): + tdLog.info("start create tables......") tsql.execute("create database if not exists %s vgroups %d wal_retention_period 3600"%(dbName, vgroups)) tsql.execute("use %s" %dbName) tsql.execute("create table if not exists %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) @@ -137,11 +138,11 @@ class TDTestCase: tsql.execute(sql) event.set() - tdLog.debug("complete to create database[%s], stable[%s] and %d child tables" %(dbName, stbName, ctbNum)) + tdLog.info("complete to create database[%s], stable[%s] and %d child tables" %(dbName, stbName, ctbNum)) return def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs): - tdLog.debug("start to insert data ............") + tdLog.info("start to insert data ............") tsql.execute("use %s" %dbName) pre_insert = "insert into " sql = pre_insert @@ -163,7 +164,7 @@ class TDTestCase: if sql != pre_insert: #print("insert sql:%s"%sql) tsql.execute(sql) - tdLog.debug("insert data ............ [OK]") + tdLog.info("insert data ............ [OK]") return def prepareEnv(self, **parameterDict): @@ -286,7 +287,7 @@ class TDTestCase: prepareEnvThread.start() tdLog.info("create topics from db") - topicName1 = 'topic_db1' + topicName1 = 'topic_db11' tdSql.execute("create topic %s as database %s" %(topicName1, parameterDict['dbName'])) consumerId = 0 From e61b51b083c67fa47c46c515d3cc86ad3df88f20 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 20 Apr 2023 17:49:40 +0800 Subject: [PATCH 023/109] fix:add test cases for escape in schemaless --- source/client/src/clientSml.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 065f2e1fdb..38c346d280 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1178,6 +1178,7 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) { terrno = 0; taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); if (terrno == TSDB_CODE_DUP_KEY) { + taosHashCleanup(kvHash); return terrno; } } @@ -1251,13 +1252,13 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat); + taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES); terrno = 0; smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags); if (terrno == TSDB_CODE_DUP_KEY) { return terrno; } smlInsertMeta(meta->colHash, meta->cols, elements->colArray); - taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES); } } uDebug("SML:0x%" PRIx64 " smlParseLineBottom end, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); From 6f595fb5fd152772c534b79f59f7abc848f0b692 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 20 Apr 2023 18:50:44 +0800 Subject: [PATCH 024/109] other: add some logs. --- source/dnode/vnode/src/tq/tqUtil.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 5ac747947f..438e5b9678 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -66,11 +66,12 @@ void initOffsetForAllRestoreTasks(STQ* pTq) { SStreamTask* pTask = *(SStreamTask**)pIter; if (pTask->taskLevel != TASK_LEVEL__SOURCE) { + tqDebug("s-task:%s not source task, not register offset", pTask->id.idStr); continue; } if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) { - tqDebug("s-task:%s skip push data, since not ready, status %d", pTask->id.idStr, pTask->status.taskStatus); + tqDebug("s-task:%s no need to record the offset, status %d", pTask->id.idStr, pTask->status.taskStatus); continue; } From f9801ba9c54235cb9bd0c1b57d506c6e8f9388c7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 20 Apr 2023 18:58:07 +0800 Subject: [PATCH 025/109] enh(stream): stop stream asap. --- source/dnode/mgmt/mgmt_vnode/src/vmInt.c | 2 +- source/dnode/vnode/src/inc/tq.h | 1 + source/dnode/vnode/src/inc/vnodeInt.h | 1 + source/dnode/vnode/src/tq/tq.c | 23 +++++++++++++++++++ source/libs/executor/src/timewindowoperator.c | 5 ++++ 5 files changed, 31 insertions(+), 1 deletion(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 0244a4fd6e..16e7ffc536 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -119,6 +119,7 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal) pVnode->pFetchQ->threadId); while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10); + tqNotifyClose(pVnode->pImpl->pTq); dInfo("vgId:%d, wait for vnode stream queue:%p is empty", pVnode->vgId, pVnode->pStreamQ); while (!taosQueueEmpty(pVnode->pStreamQ)) taosMsleep(10); @@ -141,7 +142,6 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal) dInfo("vgId:%d, vnode is closed", pVnode->vgId); if (commitAndRemoveWal) { - char path[TSDB_FILENAME_LEN] = {0}; snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d%swal", TD_DIRSEP, pVnode->vgId, TD_DIRSEP); dInfo("vgId:%d, remove all wals, path:%s", pVnode->vgId, path); tfsRmdir(pMgmt->pTfs, path); diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index db17e4f533..fdd1ece41a 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -121,6 +121,7 @@ struct STQ { TTB* pExecStore; TTB* pCheckStore; SStreamMeta* pStreamMeta; + bool closing; }; typedef struct { diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 81f7c3d52a..416bc6cdc7 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -190,6 +190,7 @@ int32_t tsdbSetKeepCfg(STsdb* pTsdb, STsdbCfg* pCfg); int tqInit(); void tqCleanUp(); STQ* tqOpen(const char* path, SVnode* pVnode); +void tqNotifyClose(STQ*); void tqClose(STQ*); int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver); int tqRegisterPushHandle(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg, SMqDataRsp* pDataRsp, diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 962b89732e..1a7af742ba 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -97,6 +97,7 @@ STQ* tqOpen(const char* path, SVnode* pVnode) { return NULL; } + pTq->closing = false; pTq->path = taosStrdup(path); pTq->pVnode = pVnode; pTq->walLogLastVer = pVnode->pWal->vers.lastVer; @@ -154,6 +155,28 @@ void tqClose(STQ* pTq) { taosMemoryFree(pTq); } +void tqNotifyClose(STQ* pTq) { + if (pTq != NULL) { + pTq->closing = true; + taosWLockLatch(&pTq->pStreamMeta->lock); + + void* pIter = NULL; + while (1) { + pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter); + if (pIter == NULL) { + break; + } + + SStreamTask* pTask = *(SStreamTask**)pIter; + tqDebug("vgId:%d s-task:%s set dropping flag", pTq->pStreamMeta->vgId, pTask->id.idStr); + pTask->status.taskStatus = TASK_STATUS__DROPPING; + qKillTask(pTask->exec.pExecutor, TSDB_CODE_SUCCESS); + } + + taosWUnLockLatch(&pTq->pStreamMeta->lock); + } +} + static int32_t doSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId, int32_t type) { int32_t len = 0; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 007a6f63d1..ef9dc779d9 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2541,6 +2541,10 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { } while (1) { + if (isTaskKilled(pTaskInfo)) { + T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); + } + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { pOperator->status = OP_RES_TO_RETURN; @@ -2635,6 +2639,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { while ((pIte = tSimpleHashIterate(pInfo->pUpdatedMap, pIte, &iter)) != NULL) { taosArrayPush(pInfo->pUpdated, pIte); } + tSimpleHashCleanup(pInfo->pUpdatedMap); pInfo->pUpdatedMap = NULL; taosArraySort(pInfo->pUpdated, winKeyCmprImpl); From 65fded4a28ab07c881d2c45fa582a8c08c39b64b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 20 Apr 2023 19:03:43 +0800 Subject: [PATCH 026/109] fix(stream): set the correct number of tasks. --- source/dnode/vnode/src/tq/tqRestore.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/dnode/vnode/src/tq/tqRestore.c b/source/dnode/vnode/src/tq/tqRestore.c index 56f0a80b9e..dd5da6b462 100644 --- a/source/dnode/vnode/src/tq/tqRestore.c +++ b/source/dnode/vnode/src/tq/tqRestore.c @@ -89,6 +89,8 @@ int32_t doCreateReqsByScanWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetS tqDebug("vgId:%d start to check wal to extract new submit block for %d tasks", vgId, numOfTasks); SArray* pTaskIdList = extractTaskIdList(pStreamMeta, numOfTasks); + // update the new task number + numOfTasks = taosArrayGetSize(pTaskIdList); for (int32_t i = 0; i < numOfTasks; ++i) { int32_t* pTaskId = taosArrayGet(pTaskIdList, i); SStreamTask* pTask = streamMetaAcquireTask(pStreamMeta, *pTaskId); From 5e6c06e2535fee40c7fd82286761c1437a6045ac Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 20 Apr 2023 23:21:18 +0800 Subject: [PATCH 027/109] fix(stream): fix the race condition during create new stream tasks. --- include/libs/stream/tstream.h | 2 + source/dnode/vnode/src/inc/tq.h | 7 --- source/dnode/vnode/src/tq/tq.c | 47 ++++++++---------- source/dnode/vnode/src/tq/tqRead.c | 3 ++ source/dnode/vnode/src/tq/tqRestore.c | 26 +++++----- source/dnode/vnode/src/tq/tqUtil.c | 70 --------------------------- source/libs/stream/src/streamExec.c | 13 +++-- source/libs/stream/src/streamMeta.c | 19 ++++---- 8 files changed, 56 insertions(+), 131 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index ba349e11f1..78fd9bed5d 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -273,6 +273,7 @@ typedef struct SStreamId { typedef struct SCheckpointInfo { int64_t id; int64_t version; // offset in WAL + int64_t currentVer;// current offset in WAL, not serialize it } SCheckpointInfo; typedef struct SStreamStatus { @@ -537,6 +538,7 @@ void streamTaskInputFail(SStreamTask* pTask); int32_t streamTryExec(SStreamTask* pTask); int32_t streamSchedExec(SStreamTask* pTask); int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBlock* pBlock); +bool streamTaskShouldStop(const SStreamStatus* pStatus); int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz); diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index fdd1ece41a..0b977e841c 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -121,7 +121,6 @@ struct STQ { TTB* pExecStore; TTB* pCheckStore; SStreamMeta* pStreamMeta; - bool closing; }; typedef struct { @@ -183,14 +182,8 @@ int32_t tqStreamTasksScanWal(STQ* pTq); char* createStreamTaskIdStr(int64_t streamId, int32_t taskId); void createStreamTaskOffsetKey(char* dst, uint64_t streamId, uint32_t taskId); int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueItem, int64_t ver); -int32_t launchTaskForWalBlock(SStreamTask* pTask, SFetchRet* pRet, STqOffset* pOffset); int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg); -void doSaveTaskOffset(STqOffsetStore* pOffsetStore, const char* pKey, int64_t ver); -void saveOffsetForAllTasks(STQ* pTq, int64_t ver); -void initOffsetForAllRestoreTasks(STQ* pTq); -int32_t transferToWalReadTask(SStreamMeta* pStreamMeta, SArray* pTaskList); - #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 1a7af742ba..fc5dcab807 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -97,7 +97,6 @@ STQ* tqOpen(const char* path, SVnode* pVnode) { return NULL; } - pTq->closing = false; pTq->path = taosStrdup(path); pTq->pVnode = pVnode; pTq->walLogLastVer = pVnode->pWal->vers.lastVer; @@ -157,7 +156,6 @@ void tqClose(STQ* pTq) { void tqNotifyClose(STQ* pTq) { if (pTq != NULL) { - pTq->closing = true; taosWLockLatch(&pTq->pStreamMeta->lock); void* pIter = NULL; @@ -169,8 +167,12 @@ void tqNotifyClose(STQ* pTq) { SStreamTask* pTask = *(SStreamTask**)pIter; tqDebug("vgId:%d s-task:%s set dropping flag", pTq->pStreamMeta->vgId, pTask->id.idStr); - pTask->status.taskStatus = TASK_STATUS__DROPPING; + pTask->status.taskStatus = TASK_STATUS__STOP; + + int64_t st = taosGetTimestampMs(); qKillTask(pTask->exec.pExecutor, TSDB_CODE_SUCCESS); + int64_t el = taosGetTimestampMs() - st; + tqDebug("vgId:%d s-task:%s is closed in %" PRId64 "ms", pTq->pStreamMeta->vgId, pTask->id.idStr, el); } taosWUnLockLatch(&pTq->pStreamMeta->lock); @@ -596,6 +598,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { pTask->pMsgCb = &pTq->pVnode->msgCb; pTask->pMeta = pTq->pStreamMeta; pTask->chkInfo.version = ver; + pTask->chkInfo.currentVer = ver; // expand executor if (pTask->fillHistory) { @@ -989,14 +992,21 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { int32_t* pRef = taosMemoryMalloc(sizeof(int32_t)); *pRef = 1; + taosWLockLatch(&pTq->pStreamMeta->lock); + void* pIter = NULL; while (1) { pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter); - if (pIter == NULL) break; - SStreamTask* pTask = *(SStreamTask**)pIter; - if (pTask->taskLevel != TASK_LEVEL__SOURCE) continue; + if (pIter == NULL) { + break; + } - qDebug("delete req enqueue stream task: %d, ver: %" PRId64, pTask->id.taskId, ver); + SStreamTask* pTask = *(SStreamTask**)pIter; + if (pTask->taskLevel != TASK_LEVEL__SOURCE) { + continue; + } + + qDebug("s-task:%s delete req enqueue, ver: %" PRId64, pTask->id.idStr, ver); if (!failed) { SStreamRefDataBlock* pRefBlock = taosAllocateQitem(sizeof(SStreamRefDataBlock), DEF_QITEM, 0); @@ -1006,7 +1016,7 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { atomic_add_fetch_32(pRefBlock->dataRef, 1); if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pRefBlock) < 0) { - qError("stream task input del failed, task id %d", pTask->id.taskId); + qError("s-task:%s stream task input del failed", pTask->id.idStr); atomic_sub_fetch_32(pRef, 1); taosFreeQitem(pRefBlock); @@ -1014,7 +1024,7 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { } if (streamSchedExec(pTask) < 0) { - qError("stream task launch failed, task id %d", pTask->id.taskId); + qError("s-task:%s stream task launch failed", pTask->id.idStr); continue; } @@ -1023,8 +1033,9 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { } } + taosWUnLockLatch(&pTq->pStreamMeta->lock); + int32_t ref = atomic_sub_fetch_32(pRef, 1); - /*A(ref >= 0);*/ if (ref == 0) { blockDataDestroy(pDelBlock); taosMemoryFree(pRef); @@ -1055,23 +1066,9 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { } blockDataDestroy(pDelBlock); #endif - return 0; } -static int32_t addSubmitBlockNLaunchTask(STqOffsetStore* pOffsetStore, SStreamTask* pTask, SStreamDataSubmit2* pSubmit, - const char* key, int64_t ver) { - doSaveTaskOffset(pOffsetStore, key, ver); - int32_t code = tqAddInputBlockNLaunchTask(pTask, (SStreamQueueItem*)pSubmit, ver); - - // remove the offset, if all functions are completed successfully. - if (code == TSDB_CODE_SUCCESS) { - tqOffsetDelete(pOffsetStore, key); - } - - return code; -} - int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) { #if 0 void* pIter = NULL; @@ -1333,8 +1330,6 @@ int32_t tqStartStreamTasks(STQ* pTq) { } tqDebug("vgId:%d start wal scan stream tasks, tasks:%d", vgId, numOfTasks); - initOffsetForAllRestoreTasks(pTq); - pRunReq->head.vgId = vgId; pRunReq->streamId = 0; pRunReq->taskId = WAL_READ_TASKS_ID; diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 25ab7209d2..2cda12c0e1 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -1023,6 +1023,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { } // update the table list handle for each stream scanner/wal reader + taosWLockLatch(&pTq->pStreamMeta->lock); while (1) { pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter); if (pIter == NULL) { @@ -1039,5 +1040,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { } } + taosWUnLockLatch(&pTq->pStreamMeta->lock); + return 0; } diff --git a/source/dnode/vnode/src/tq/tqRestore.c b/source/dnode/vnode/src/tq/tqRestore.c index dd5da6b462..741e633ae4 100644 --- a/source/dnode/vnode/src/tq/tqRestore.c +++ b/source/dnode/vnode/src/tq/tqRestore.c @@ -15,12 +15,12 @@ #include "tq.h" -static int32_t doCreateReqsByScanWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetStore, bool* pScanIdle); +static int32_t doCreateReqsByScanWal(SStreamMeta* pStreamMeta, bool* pScanIdle); // this function should be executed by stream threads. // there is a case that the WAL increases more fast than the restore procedure, and this restore procedure // will not stop eventually. -int tqStreamTasksScanWal(STQ* pTq) { +int32_t tqStreamTasksScanWal(STQ* pTq) { int32_t vgId = TD_VID(pTq->pVnode); SStreamMeta* pMeta = pTq->pStreamMeta; int64_t st = taosGetTimestampMs(); @@ -31,7 +31,7 @@ int tqStreamTasksScanWal(STQ* pTq) { // check all restore tasks bool shouldIdle = true; - doCreateReqsByScanWal(pTq->pStreamMeta, pTq->pOffsetStore, &shouldIdle); + doCreateReqsByScanWal(pTq->pStreamMeta, &shouldIdle); int32_t times = 0; @@ -76,7 +76,7 @@ static SArray* extractTaskIdList(SStreamMeta* pStreamMeta, int32_t numOfTasks) { return pTaskIdList; } -int32_t doCreateReqsByScanWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetStore, bool* pScanIdle) { +int32_t doCreateReqsByScanWal(SStreamMeta* pStreamMeta, bool* pScanIdle) { *pScanIdle = true; bool noNewDataInWal = true; int32_t vgId = pStreamMeta->vgId; @@ -99,12 +99,14 @@ int32_t doCreateReqsByScanWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetS } int32_t status = pTask->status.taskStatus; - if ((pTask->taskLevel != TASK_LEVEL__SOURCE) || (status == TASK_STATUS__DROPPING)) { + if (pTask->taskLevel != TASK_LEVEL__SOURCE) { + tqDebug("s-task:%s not source task, no need to start", pTask->id.idStr); streamMetaReleaseTask(pStreamMeta, pTask); continue; } - if (status == TASK_STATUS__RECOVER_PREPARE || status == TASK_STATUS__WAIT_DOWNSTREAM) { + if (streamTaskShouldStop(&pTask->status) || status == TASK_STATUS__RECOVER_PREPARE || + status == TASK_STATUS__WAIT_DOWNSTREAM) { tqDebug("s-task:%s skip push data, not ready for processing, status %d", pTask->id.idStr, status); streamMetaReleaseTask(pStreamMeta, pTask); continue; @@ -122,19 +124,15 @@ int32_t doCreateReqsByScanWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetS *pScanIdle = false; - // check if offset value exists - STqOffset* pOffset = tqOffsetRead(pOffsetStore, key); - ASSERT(pOffset != NULL); - // seek the stored version and extract data from WAL - int32_t code = walReadSeekVer(pTask->exec.pWalReader, pOffset->val.version); + int32_t code = walReadSeekVer(pTask->exec.pWalReader, pTask->chkInfo.currentVer); if (code != TSDB_CODE_SUCCESS) { // no data in wal, quit streamMetaReleaseTask(pStreamMeta, pTask); continue; } // append the data for the stream - tqDebug("vgId:%d wal reader seek to ver:%" PRId64 " %s", vgId, pOffset->val.version, pTask->id.idStr); + tqDebug("vgId:%d wal reader seek to ver:%" PRId64 " %s", vgId, pTask->chkInfo.currentVer, pTask->id.idStr); SPackedData packData = {0}; code = extractSubmitMsgFromWal(pTask->exec.pWalReader, &packData); @@ -156,9 +154,9 @@ int32_t doCreateReqsByScanWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetS tqDebug("s-task:%s submit data extracted from WAL", pTask->id.idStr); code = tqAddInputBlockNLaunchTask(pTask, (SStreamQueueItem*)p, packData.ver); if (code == TSDB_CODE_SUCCESS) { - pOffset->val.version = walReaderGetCurrentVer(pTask->exec.pWalReader); + pTask->chkInfo.currentVer = walReaderGetCurrentVer(pTask->exec.pWalReader); tqDebug("s-task:%s set the ver:%" PRId64 " from WALReader after extract block from WAL", pTask->id.idStr, - pOffset->val.version); + pTask->chkInfo.currentVer); } else { // do nothing } diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 438e5b9678..ad37ac329b 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -55,76 +55,6 @@ int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueI return TSDB_CODE_SUCCESS; } -void initOffsetForAllRestoreTasks(STQ* pTq) { - void* pIter = NULL; - - while(1) { - pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter); - if (pIter == NULL) { - break; - } - - SStreamTask* pTask = *(SStreamTask**)pIter; - if (pTask->taskLevel != TASK_LEVEL__SOURCE) { - tqDebug("s-task:%s not source task, not register offset", pTask->id.idStr); - continue; - } - - if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) { - tqDebug("s-task:%s no need to record the offset, status %d", pTask->id.idStr, pTask->status.taskStatus); - continue; - } - - char key[128] = {0}; - createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId); - - STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, key); - if (pOffset == NULL) { - doSaveTaskOffset(pTq->pOffsetStore, key, pTask->chkInfo.version); - } - } -} - -void saveOffsetForAllTasks(STQ* pTq, int64_t ver) { - void* pIter = NULL; - - while(1) { - pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter); - if (pIter == NULL) { - break; - } - - SStreamTask* pTask = *(SStreamTask**)pIter; - if (pTask->taskLevel != TASK_LEVEL__SOURCE) { - continue; - } - - if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) { - tqDebug("s-task:%s skip push data, not ready for processing, status %d", pTask->id.idStr, - pTask->status.taskStatus); - continue; - } - - char key[128] = {0}; - createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId); - - STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, key); - if (pOffset == NULL) { - doSaveTaskOffset(pTq->pOffsetStore, key, ver); - } - } -} - -void doSaveTaskOffset(STqOffsetStore* pOffsetStore, const char* pKey, int64_t ver) { - STqOffset offset = {0}; - tqOffsetResetToLog(&offset.val, ver); - - tstrncpy(offset.subKey, pKey, tListLen(offset.subKey)); - - // keep the offset info in the offset store - tqOffsetWrite(pOffsetStore, &offset); -} - static int32_t tqInitDataRsp(SMqDataRsp* pRsp, const SMqPollReq* pReq, int8_t subType) { pRsp->reqOffset = pReq->reqOffset; diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index f52af66387..f82e9b9621 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -17,6 +17,11 @@ #define STREAM_EXEC_MAX_BATCH_NUM 100 +bool streamTaskShouldStop(const SStreamStatus* pStatus) { + int32_t status = atomic_load_8((int8_t*) &pStatus->taskStatus); + return (status == TASK_STATUS__STOP) || (status == TASK_STATUS__DROPPING); +} + static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* pRes) { int32_t code = TSDB_CODE_SUCCESS; void* pExecutor = pTask->exec.pExecutor; @@ -66,7 +71,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* // pExecutor while (1) { - if (pTask->status.taskStatus == TASK_STATUS__DROPPING) { + if (streamTaskShouldStop(&pTask->status)) { return 0; } @@ -134,7 +139,7 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { int32_t batchCnt = 0; while (1) { - if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) { + if (streamTaskShouldStop(&pTask->status)) { taosArrayDestroy(pRes); return 0; } @@ -270,7 +275,7 @@ int32_t streamExecForAll(SStreamTask* pTask) { } } - if (pTask->status.taskStatus == TASK_STATUS__DROPPING) { + if (streamTaskShouldStop(&pTask->status)) { if (pInput) { streamFreeQitem(pInput); } @@ -368,7 +373,7 @@ int32_t streamTryExec(SStreamTask* pTask) { atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); qDebug("s-task:%s exec completed", pTask->id.idStr); - if (!taosQueueEmpty(pTask->inputQueue->queue) && (pTask->status.taskStatus != TASK_STATUS__DROPPING)) { + if (!taosQueueEmpty(pTask->inputQueue->queue) && (!streamTaskShouldStop(&pTask->status))) { streamSchedExec(pTask); } } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 90c06dbd69..7f06892a9f 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -191,10 +191,12 @@ SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId) { taosRLockLatch(&pMeta->lock); SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t)); - if (ppTask != NULL && (atomic_load_8(&((*ppTask)->status.taskStatus)) != TASK_STATUS__DROPPING)) { - atomic_add_fetch_32(&(*ppTask)->refCnt, 1); - taosRUnLockLatch(&pMeta->lock); - return *ppTask; + if (ppTask != NULL) { + if (streamTaskShouldStop(&(*ppTask)->status)) { + atomic_add_fetch_32(&(*ppTask)->refCnt, 1); + taosRUnLockLatch(&pMeta->lock); + return *ppTask; + } } taosRUnLockLatch(&pMeta->lock); @@ -205,7 +207,7 @@ void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask) { int32_t left = atomic_sub_fetch_32(&pTask->refCnt, 1); ASSERT(left >= 0); if (left == 0) { - ASSERT(atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING); + ASSERT(streamTaskShouldStop(&pTask->status)); tFreeStreamTask(pTask); } } @@ -216,11 +218,8 @@ void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) { SStreamTask* pTask = *ppTask; taosHashRemove(pMeta->pTasks, &taskId, sizeof(int32_t)); tdbTbDelete(pMeta->pTaskDb, &taskId, sizeof(int32_t), pMeta->txn); - /*if (pTask->timer) { - * taosTmrStop(pTask->timer);*/ - /*pTask->timer = NULL;*/ - /*}*/ - atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__DROPPING); + + atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__STOP); taosWLockLatch(&pMeta->lock); streamMetaReleaseTask(pMeta, pTask); From 471abd9160ce8b6d890b4a408a91ac14df86b5ba Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 20 Apr 2023 23:39:13 +0800 Subject: [PATCH 028/109] log: update the log. --- source/dnode/vnode/src/tq/tq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index fc5dcab807..8634029dd8 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1016,7 +1016,7 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { atomic_add_fetch_32(pRefBlock->dataRef, 1); if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pRefBlock) < 0) { - qError("s-task:%s stream task input del failed", pTask->id.idStr); + qError("s-task:%s stream task append submit into input queue failed", pTask->id.idStr); atomic_sub_fetch_32(pRef, 1); taosFreeQitem(pRefBlock); From e5d0f2decc8d02fc332e4c34e45a316ac2558923 Mon Sep 17 00:00:00 2001 From: xiaolei li <85657333+xleili@users.noreply.github.com> Date: Fri, 21 Apr 2023 09:03:48 +0800 Subject: [PATCH 029/109] enhance: enterprise package include jdbc driver (#21001) --- packaging/tools/makeclient.sh | 3 ++- packaging/tools/makepkg.sh | 15 ++++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index b473f3b527..28dc770755 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -197,7 +197,8 @@ if [[ $productName == "TDengine" ]]; then mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then if [ "$osType" != "Darwin" ]; then - [ -f ${build_dir}/lib/*.jar ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || : + jars=$(ls ${build_dir}/lib/*.jar 2>/dev/null|wc -l) + [ "${jars}" != "0" ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || : fi git clone --depth 1 https://github.com/taosdata/driver-go ${install_dir}/connector/go rm -rf ${install_dir}/connector/go/.git ||: diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index e4df233d67..a590835257 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -338,7 +338,20 @@ if [ "$verMode" == "cluster" ] || [ "$verMode" == "cloud" ]; then connector_dir="${code_dir}/connector" mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then - [ -f ${build_dir}/lib/*.jar ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || : + tmp_pwd=`pwd` + cd ${install_dir}/connector + if [ ! -d taos-connector-jdbc ];then + git clone -b main --depth=1 https://github.com/taosdata/taos-connector-jdbc.git ||: + fi + cd taos-connector-jdbc + mvn clean package -Dmaven.test.skip=true + echo ${build_dir}/lib/ + cp target/*.jar ${build_dir}/lib/ + cd ${install_dir}/connector + rm -rf taos-connector-jdbc + cd ${tmp_pwd} + jars=$(ls ${build_dir}/lib/*.jar 2>/dev/null|wc -l) + [ "${jars}" != "0" ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || : git clone --depth 1 https://github.com/taosdata/driver-go ${install_dir}/connector/go rm -rf ${install_dir}/connector/go/.git ||: From 6c641cff39e7cb4d7947953e6ae4870be423e6df Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 21 Apr 2023 09:37:16 +0800 Subject: [PATCH 030/109] fix(stream): fix error in start stream tasks. --- source/dnode/vnode/src/tq/tq.c | 2 -- source/dnode/vnode/src/tq/tqRestore.c | 6 +++--- source/libs/stream/src/streamMeta.c | 2 +- tests/script/tsim/stream/basic1.sim | 2 +- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 8634029dd8..ae52db163f 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1016,8 +1016,6 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { atomic_add_fetch_32(pRefBlock->dataRef, 1); if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pRefBlock) < 0) { - qError("s-task:%s stream task append submit into input queue failed", pTask->id.idStr); - atomic_sub_fetch_32(pRef, 1); taosFreeQitem(pRefBlock); continue; diff --git a/source/dnode/vnode/src/tq/tqRestore.c b/source/dnode/vnode/src/tq/tqRestore.c index 741e633ae4..20bbcf2eeb 100644 --- a/source/dnode/vnode/src/tq/tqRestore.c +++ b/source/dnode/vnode/src/tq/tqRestore.c @@ -15,7 +15,7 @@ #include "tq.h" -static int32_t doCreateReqsByScanWal(SStreamMeta* pStreamMeta, bool* pScanIdle); +static int32_t createStreamRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle); // this function should be executed by stream threads. // there is a case that the WAL increases more fast than the restore procedure, and this restore procedure @@ -31,7 +31,7 @@ int32_t tqStreamTasksScanWal(STQ* pTq) { // check all restore tasks bool shouldIdle = true; - doCreateReqsByScanWal(pTq->pStreamMeta, &shouldIdle); + createStreamRunReq(pTq->pStreamMeta, &shouldIdle); int32_t times = 0; @@ -76,7 +76,7 @@ static SArray* extractTaskIdList(SStreamMeta* pStreamMeta, int32_t numOfTasks) { return pTaskIdList; } -int32_t doCreateReqsByScanWal(SStreamMeta* pStreamMeta, bool* pScanIdle) { +int32_t createStreamRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle) { *pScanIdle = true; bool noNewDataInWal = true; int32_t vgId = pStreamMeta->vgId; diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 7f06892a9f..065e9d280f 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -192,7 +192,7 @@ SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId) { SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t)); if (ppTask != NULL) { - if (streamTaskShouldStop(&(*ppTask)->status)) { + if (!streamTaskShouldStop(&(*ppTask)->status)) { atomic_add_fetch_32(&(*ppTask)->refCnt, 1); taosRUnLockLatch(&pMeta->lock); return *ppTask; diff --git a/tests/script/tsim/stream/basic1.sim b/tests/script/tsim/stream/basic1.sim index 15ca6bf7c9..65032817b3 100644 --- a/tests/script/tsim/stream/basic1.sim +++ b/tests/script/tsim/stream/basic1.sim @@ -37,7 +37,7 @@ if $loop_count == 20 then endi if $rows != 4 then - print =====rows=$rows, expect 4 + print =====rows=$rows expect 4 goto loop0 endi From 8fac91e26520492ee44208d2130f6162b908462b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 21 Apr 2023 09:38:51 +0800 Subject: [PATCH 031/109] refactor: do some internal refactor. --- source/dnode/vnode/src/inc/tq.h | 1 - source/dnode/vnode/src/tq/tqRestore.c | 4 ---- source/dnode/vnode/src/tq/tqUtil.c | 15 --------------- 3 files changed, 20 deletions(-) diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 0b977e841c..acc0d29382 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -180,7 +180,6 @@ int32_t tqStreamTasksScanWal(STQ* pTq); // tq util char* createStreamTaskIdStr(int64_t streamId, int32_t taskId); -void createStreamTaskOffsetKey(char* dst, uint64_t streamId, uint32_t taskId); int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueItem, int64_t ver); int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg); diff --git a/source/dnode/vnode/src/tq/tqRestore.c b/source/dnode/vnode/src/tq/tqRestore.c index 20bbcf2eeb..6081057a4b 100644 --- a/source/dnode/vnode/src/tq/tqRestore.c +++ b/source/dnode/vnode/src/tq/tqRestore.c @@ -112,10 +112,6 @@ int32_t createStreamRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle) { continue; } - // check if offset value exists - char key[128] = {0}; - createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId); - if (tInputQueueIsFull(pTask)) { tqDebug("vgId:%d s-task:%s input queue is full, do nothing", vgId, pTask->id.idStr); streamMetaReleaseTask(pStreamMeta, pTask); diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index ad37ac329b..017b479c1b 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -25,21 +25,6 @@ char* createStreamTaskIdStr(int64_t streamId, int32_t taskId) { return taosStrdup(buf); } -// stream_task:stream_id:task_id -void createStreamTaskOffsetKey(char* dst, uint64_t streamId, uint32_t taskId) { - int32_t n = 12; - char* p = dst; - - memcpy(p, "stream_task:", n); - p += n; - - int32_t inc = tintToHex(streamId, p); - p += inc; - - *(p++) = ':'; - tintToHex(taskId, p); -} - int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueItem, int64_t ver) { int32_t code = tAppendDataToInputQueue(pTask, pQueueItem); if (code < 0) { From e4199a62defae442067bc2031ca1dc0d0e14cc9c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 21 Apr 2023 10:37:50 +0800 Subject: [PATCH 032/109] fix(stream): fix memory leak. --- source/libs/executor/src/timewindowoperator.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index ef9dc779d9..be2bd0e6e2 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2542,6 +2542,16 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { while (1) { if (isTaskKilled(pTaskInfo)) { + + if (pInfo->pUpdated != NULL) { + pInfo->pUpdated = taosArrayDestroy(pInfo->pUpdated); + } + + if (pInfo->pUpdatedMap != NULL) { + tSimpleHashCleanup(pInfo->pUpdatedMap); + pInfo->pUpdatedMap = NULL; + } + T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } From 3d8faa690d6f7f6b13cfa26316c8562b0edb3a02 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 21 Apr 2023 11:39:02 +0800 Subject: [PATCH 033/109] fix:[TS-3250] change strtegy in schemaless if modifyDBSchema error --- source/client/inc/clientSml.h | 2 +- source/client/src/clientSml.c | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index 92896e6f23..43a7e8ba45 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -70,7 +70,7 @@ extern "C" { #define VALUE_LEN 6 #define OTD_JSON_FIELDS_NUM 4 -#define MAX_RETRY_TIMES 100 +#define MAX_RETRY_TIMES 10 typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType; typedef enum { diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 17150286e1..763d4b6915 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -535,7 +535,7 @@ static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSm if (index) { if (colField[*index].type != kv->type) { uError("SML:0x%" PRIx64 " point type and db type mismatch. point type: %d, db type: %d, key: %s", info->id, colField[*index].type, kv->type, kv->key); - return TSDB_CODE_TSC_INVALID_VALUE; + return TSDB_CODE_SML_INVALID_DATA; } if ((colField[*index].type == TSDB_DATA_TYPE_VARCHAR && @@ -1494,8 +1494,8 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL do { code = smlModifyDBSchemas(info); - if (code == 0) break; - taosMsleep(500); + if (code == 0 || code == TSDB_CODE_SML_INVALID_DATA) break; + taosMsleep(100); uInfo("SML:0x%" PRIx64 " smlModifyDBSchemas retry code:%s, times:%d", info->id, tstrerror(code), retryNum); } while (retryNum++ < taosHashGetSize(info->superTables) * MAX_RETRY_TIMES); From 81b0670bd5cb4044f8920f735c25dc17c5c4dcdf Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 21 Apr 2023 11:43:39 +0800 Subject: [PATCH 034/109] fix:[TS-3082] change offset to firstver if offset is smller than firstVer when wal is removed --- source/libs/executor/src/executor.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 1732ec04a7..546cd18cda 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1108,6 +1108,11 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT pScanBaseInfo->dataReader = NULL; // let's seek to the next version in wal file + int64_t firstVer = walGetFirstVer(pInfo->tqReader->pWalReader->pWal); + if (pOffset->version + 1 < firstVer){ + pOffset->version = firstVer - 1; + } + if (tqSeekVer(pInfo->tqReader, pOffset->version + 1, id) < 0) { qError("tqSeekVer failed ver:%" PRId64 ", %s", pOffset->version + 1, id); return -1; From b6f4cac619091e07b8a67d7fb50106ddf35ca110 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 21 Apr 2023 13:35:24 +0800 Subject: [PATCH 035/109] fix(stream): set the correct offset version. --- source/dnode/vnode/src/tq/tqRestore.c | 5 ++--- source/libs/stream/src/streamExec.c | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqRestore.c b/source/dnode/vnode/src/tq/tqRestore.c index 6081057a4b..c164d037e0 100644 --- a/source/dnode/vnode/src/tq/tqRestore.c +++ b/source/dnode/vnode/src/tq/tqRestore.c @@ -128,7 +128,7 @@ int32_t createStreamRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle) { } // append the data for the stream - tqDebug("vgId:%d wal reader seek to ver:%" PRId64 " %s", vgId, pTask->chkInfo.currentVer, pTask->id.idStr); + tqDebug("vgId:%d s-task:%s wal reader seek to ver:%" PRId64, vgId, pTask->id.idStr, pTask->chkInfo.currentVer); SPackedData packData = {0}; code = extractSubmitMsgFromWal(pTask->exec.pWalReader, &packData); @@ -147,14 +147,13 @@ int32_t createStreamRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle) { noNewDataInWal = false; - tqDebug("s-task:%s submit data extracted from WAL", pTask->id.idStr); code = tqAddInputBlockNLaunchTask(pTask, (SStreamQueueItem*)p, packData.ver); if (code == TSDB_CODE_SUCCESS) { pTask->chkInfo.currentVer = walReaderGetCurrentVer(pTask->exec.pWalReader); tqDebug("s-task:%s set the ver:%" PRId64 " from WALReader after extract block from WAL", pTask->id.idStr, pTask->chkInfo.currentVer); } else { - // do nothing + tqError("s-task:%s append input queue failed, ver:%"PRId64, pTask->id.idStr, pTask->chkInfo.currentVer); } streamDataSubmitDestroy(p); diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index f82e9b9621..325d315262 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -111,7 +111,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* continue; } - qDebug("task %d(child %d) executed and get block", pTask->id.taskId, pTask->selfChildId); + qDebug("s-task:%s (child %d) executed and get block", pTask->id.idStr, pTask->selfChildId); SSDataBlock block = {0}; assignOneDataBlock(&block, output); @@ -306,7 +306,7 @@ int32_t streamExecForAll(SStreamTask* pTask) { ", checkPoint id:%" PRId64 " -> %" PRId64, pTask->id.idStr, pTask->chkInfo.version, dataVer, pTask->chkInfo.id, ckId); - pTask->chkInfo = (SCheckpointInfo) {.version = dataVer, .id = ckId}; + pTask->chkInfo = (SCheckpointInfo) {.version = dataVer, .id = ckId, .currentVer = pTask->chkInfo.currentVer}; taosWLockLatch(&pTask->pMeta->lock); streamMetaSaveTask(pTask->pMeta, pTask); From 3c2fc48e7424b794acea3b6567fa9d40fe19f665 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 21 Apr 2023 14:17:07 +0800 Subject: [PATCH 036/109] feat: support new table_prefix/table_suffix mode --- include/common/ttokendef.h | 659 ++--- include/util/tdef.h | 8 +- include/util/tutil.h | 18 +- source/dnode/mnode/impl/src/mndDb.c | 10 +- source/libs/command/src/command.c | 7 +- source/libs/parser/inc/sql.y | 4 +- source/libs/parser/src/parAstCreater.c | 22 +- source/libs/parser/src/parTranslater.c | 33 +- source/libs/parser/src/sql.c | 3518 ++++++++++++++---------- 9 files changed, 2477 insertions(+), 1802 deletions(-) mode change 100644 => 100755 source/libs/parser/inc/sql.y diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index 641cbbb588..10b5328e6d 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -16,335 +16,336 @@ #ifndef _TD_COMMON_TOKEN_H_ #define _TD_COMMON_TOKEN_H_ -#define TK_OR 1 -#define TK_AND 2 -#define TK_UNION 3 -#define TK_ALL 4 -#define TK_MINUS 5 -#define TK_EXCEPT 6 -#define TK_INTERSECT 7 -#define TK_NK_BITAND 8 -#define TK_NK_BITOR 9 -#define TK_NK_LSHIFT 10 -#define TK_NK_RSHIFT 11 -#define TK_NK_PLUS 12 -#define TK_NK_MINUS 13 -#define TK_NK_STAR 14 -#define TK_NK_SLASH 15 -#define TK_NK_REM 16 -#define TK_NK_CONCAT 17 -#define TK_CREATE 18 -#define TK_ACCOUNT 19 -#define TK_NK_ID 20 -#define TK_PASS 21 -#define TK_NK_STRING 22 -#define TK_ALTER 23 -#define TK_PPS 24 -#define TK_TSERIES 25 -#define TK_STORAGE 26 -#define TK_STREAMS 27 -#define TK_QTIME 28 -#define TK_DBS 29 -#define TK_USERS 30 -#define TK_CONNS 31 -#define TK_STATE 32 -#define TK_USER 33 -#define TK_ENABLE 34 -#define TK_NK_INTEGER 35 -#define TK_SYSINFO 36 -#define TK_DROP 37 -#define TK_GRANT 38 -#define TK_ON 39 -#define TK_TO 40 -#define TK_REVOKE 41 -#define TK_FROM 42 -#define TK_SUBSCRIBE 43 -#define TK_NK_COMMA 44 -#define TK_READ 45 -#define TK_WRITE 46 -#define TK_NK_DOT 47 -#define TK_WITH 48 -#define TK_DNODE 49 -#define TK_PORT 50 -#define TK_DNODES 51 -#define TK_NK_IPTOKEN 52 -#define TK_FORCE 53 -#define TK_LOCAL 54 -#define TK_QNODE 55 -#define TK_BNODE 56 -#define TK_SNODE 57 -#define TK_MNODE 58 -#define TK_DATABASE 59 -#define TK_USE 60 -#define TK_FLUSH 61 -#define TK_TRIM 62 -#define TK_COMPACT 63 -#define TK_IF 64 -#define TK_NOT 65 -#define TK_EXISTS 66 -#define TK_BUFFER 67 -#define TK_CACHEMODEL 68 -#define TK_CACHESIZE 69 -#define TK_COMP 70 -#define TK_DURATION 71 -#define TK_NK_VARIABLE 72 -#define TK_MAXROWS 73 -#define TK_MINROWS 74 -#define TK_KEEP 75 -#define TK_PAGES 76 -#define TK_PAGESIZE 77 -#define TK_TSDB_PAGESIZE 78 -#define TK_PRECISION 79 -#define TK_REPLICA 80 -#define TK_VGROUPS 81 -#define TK_SINGLE_STABLE 82 -#define TK_RETENTIONS 83 -#define TK_SCHEMALESS 84 -#define TK_WAL_LEVEL 85 -#define TK_WAL_FSYNC_PERIOD 86 -#define TK_WAL_RETENTION_PERIOD 87 -#define TK_WAL_RETENTION_SIZE 88 -#define TK_WAL_ROLL_PERIOD 89 -#define TK_WAL_SEGMENT_SIZE 90 -#define TK_STT_TRIGGER 91 -#define TK_TABLE_PREFIX 92 -#define TK_TABLE_SUFFIX 93 -#define TK_NK_COLON 94 -#define TK_MAX_SPEED 95 -#define TK_START 96 -#define TK_TIMESTAMP 97 -#define TK_END 98 -#define TK_TABLE 99 -#define TK_NK_LP 100 -#define TK_NK_RP 101 -#define TK_STABLE 102 -#define TK_ADD 103 -#define TK_COLUMN 104 -#define TK_MODIFY 105 -#define TK_RENAME 106 -#define TK_TAG 107 -#define TK_SET 108 -#define TK_NK_EQ 109 -#define TK_USING 110 -#define TK_TAGS 111 -#define TK_BOOL 112 -#define TK_TINYINT 113 -#define TK_SMALLINT 114 -#define TK_INT 115 -#define TK_INTEGER 116 -#define TK_BIGINT 117 -#define TK_FLOAT 118 -#define TK_DOUBLE 119 -#define TK_BINARY 120 -#define TK_NCHAR 121 -#define TK_UNSIGNED 122 -#define TK_JSON 123 -#define TK_VARCHAR 124 -#define TK_MEDIUMBLOB 125 -#define TK_BLOB 126 -#define TK_VARBINARY 127 -#define TK_DECIMAL 128 -#define TK_COMMENT 129 -#define TK_MAX_DELAY 130 -#define TK_WATERMARK 131 -#define TK_ROLLUP 132 -#define TK_TTL 133 -#define TK_SMA 134 -#define TK_DELETE_MARK 135 -#define TK_FIRST 136 -#define TK_LAST 137 -#define TK_SHOW 138 -#define TK_PRIVILEGES 139 -#define TK_DATABASES 140 -#define TK_TABLES 141 -#define TK_STABLES 142 -#define TK_MNODES 143 -#define TK_QNODES 144 -#define TK_FUNCTIONS 145 -#define TK_INDEXES 146 -#define TK_ACCOUNTS 147 -#define TK_APPS 148 -#define TK_CONNECTIONS 149 -#define TK_LICENCES 150 -#define TK_GRANTS 151 -#define TK_QUERIES 152 -#define TK_SCORES 153 -#define TK_TOPICS 154 -#define TK_VARIABLES 155 -#define TK_CLUSTER 156 -#define TK_BNODES 157 -#define TK_SNODES 158 -#define TK_TRANSACTIONS 159 -#define TK_DISTRIBUTED 160 -#define TK_CONSUMERS 161 -#define TK_SUBSCRIPTIONS 162 -#define TK_VNODES 163 -#define TK_ALIVE 164 -#define TK_LIKE 165 -#define TK_TBNAME 166 -#define TK_QTAGS 167 -#define TK_AS 168 -#define TK_INDEX 169 -#define TK_FUNCTION 170 -#define TK_INTERVAL 171 -#define TK_COUNT 172 -#define TK_LAST_ROW 173 -#define TK_TOPIC 174 -#define TK_META 175 -#define TK_CONSUMER 176 -#define TK_GROUP 177 -#define TK_DESC 178 -#define TK_DESCRIBE 179 -#define TK_RESET 180 -#define TK_QUERY 181 -#define TK_CACHE 182 -#define TK_EXPLAIN 183 -#define TK_ANALYZE 184 -#define TK_VERBOSE 185 -#define TK_NK_BOOL 186 -#define TK_RATIO 187 -#define TK_NK_FLOAT 188 -#define TK_OUTPUTTYPE 189 -#define TK_AGGREGATE 190 -#define TK_BUFSIZE 191 -#define TK_LANGUAGE 192 -#define TK_REPLACE 193 -#define TK_STREAM 194 -#define TK_INTO 195 -#define TK_TRIGGER 196 -#define TK_AT_ONCE 197 -#define TK_WINDOW_CLOSE 198 -#define TK_IGNORE 199 -#define TK_EXPIRED 200 -#define TK_FILL_HISTORY 201 -#define TK_UPDATE 202 -#define TK_SUBTABLE 203 -#define TK_KILL 204 -#define TK_CONNECTION 205 -#define TK_TRANSACTION 206 -#define TK_BALANCE 207 -#define TK_VGROUP 208 -#define TK_LEADER 209 -#define TK_MERGE 210 -#define TK_REDISTRIBUTE 211 -#define TK_SPLIT 212 -#define TK_DELETE 213 -#define TK_INSERT 214 -#define TK_NULL 215 -#define TK_NK_QUESTION 216 -#define TK_NK_ARROW 217 -#define TK_ROWTS 218 -#define TK_QSTART 219 -#define TK_QEND 220 -#define TK_QDURATION 221 -#define TK_WSTART 222 -#define TK_WEND 223 -#define TK_WDURATION 224 -#define TK_IROWTS 225 -#define TK_ISFILLED 226 -#define TK_CAST 227 -#define TK_NOW 228 -#define TK_TODAY 229 -#define TK_TIMEZONE 230 -#define TK_CLIENT_VERSION 231 -#define TK_SERVER_VERSION 232 -#define TK_SERVER_STATUS 233 -#define TK_CURRENT_USER 234 -#define TK_CASE 235 -#define TK_WHEN 236 -#define TK_THEN 237 -#define TK_ELSE 238 -#define TK_BETWEEN 239 -#define TK_IS 240 -#define TK_NK_LT 241 -#define TK_NK_GT 242 -#define TK_NK_LE 243 -#define TK_NK_GE 244 -#define TK_NK_NE 245 -#define TK_MATCH 246 -#define TK_NMATCH 247 -#define TK_CONTAINS 248 -#define TK_IN 249 -#define TK_JOIN 250 -#define TK_INNER 251 -#define TK_SELECT 252 -#define TK_DISTINCT 253 -#define TK_WHERE 254 -#define TK_PARTITION 255 -#define TK_BY 256 -#define TK_SESSION 257 -#define TK_STATE_WINDOW 258 -#define TK_EVENT_WINDOW 259 -#define TK_SLIDING 260 -#define TK_FILL 261 -#define TK_VALUE 262 -#define TK_VALUE_F 263 -#define TK_NONE 264 -#define TK_PREV 265 -#define TK_NULL_F 266 -#define TK_LINEAR 267 -#define TK_NEXT 268 -#define TK_HAVING 269 -#define TK_RANGE 270 -#define TK_EVERY 271 -#define TK_ORDER 272 -#define TK_SLIMIT 273 -#define TK_SOFFSET 274 -#define TK_LIMIT 275 -#define TK_OFFSET 276 -#define TK_ASC 277 -#define TK_NULLS 278 -#define TK_ABORT 279 -#define TK_AFTER 280 -#define TK_ATTACH 281 -#define TK_BEFORE 282 -#define TK_BEGIN 283 -#define TK_BITAND 284 -#define TK_BITNOT 285 -#define TK_BITOR 286 -#define TK_BLOCKS 287 -#define TK_CHANGE 288 -#define TK_COMMA 289 -#define TK_CONCAT 290 -#define TK_CONFLICT 291 -#define TK_COPY 292 -#define TK_DEFERRED 293 -#define TK_DELIMITERS 294 -#define TK_DETACH 295 -#define TK_DIVIDE 296 -#define TK_DOT 297 -#define TK_EACH 298 -#define TK_FAIL 299 -#define TK_FILE 300 -#define TK_FOR 301 -#define TK_GLOB 302 -#define TK_ID 303 -#define TK_IMMEDIATE 304 -#define TK_IMPORT 305 -#define TK_INITIALLY 306 -#define TK_INSTEAD 307 -#define TK_ISNULL 308 -#define TK_KEY 309 -#define TK_MODULES 310 -#define TK_NK_BITNOT 311 -#define TK_NK_SEMI 312 -#define TK_NOTNULL 313 -#define TK_OF 314 -#define TK_PLUS 315 -#define TK_PRIVILEGE 316 -#define TK_RAISE 317 -#define TK_RESTRICT 318 -#define TK_ROW 319 -#define TK_SEMI 320 -#define TK_STAR 321 -#define TK_STATEMENT 322 -#define TK_STRICT 323 -#define TK_STRING 324 -#define TK_TIMES 325 -#define TK_VALUES 326 -#define TK_VARIABLE 327 -#define TK_VIEW 328 -#define TK_WAL 329 +#define TK_OR 1 +#define TK_AND 2 +#define TK_UNION 3 +#define TK_ALL 4 +#define TK_MINUS 5 +#define TK_EXCEPT 6 +#define TK_INTERSECT 7 +#define TK_NK_BITAND 8 +#define TK_NK_BITOR 9 +#define TK_NK_LSHIFT 10 +#define TK_NK_RSHIFT 11 +#define TK_NK_PLUS 12 +#define TK_NK_MINUS 13 +#define TK_NK_STAR 14 +#define TK_NK_SLASH 15 +#define TK_NK_REM 16 +#define TK_NK_CONCAT 17 +#define TK_CREATE 18 +#define TK_ACCOUNT 19 +#define TK_NK_ID 20 +#define TK_PASS 21 +#define TK_NK_STRING 22 +#define TK_ALTER 23 +#define TK_PPS 24 +#define TK_TSERIES 25 +#define TK_STORAGE 26 +#define TK_STREAMS 27 +#define TK_QTIME 28 +#define TK_DBS 29 +#define TK_USERS 30 +#define TK_CONNS 31 +#define TK_STATE 32 +#define TK_USER 33 +#define TK_ENABLE 34 +#define TK_NK_INTEGER 35 +#define TK_SYSINFO 36 +#define TK_DROP 37 +#define TK_GRANT 38 +#define TK_ON 39 +#define TK_TO 40 +#define TK_REVOKE 41 +#define TK_FROM 42 +#define TK_SUBSCRIBE 43 +#define TK_NK_COMMA 44 +#define TK_READ 45 +#define TK_WRITE 46 +#define TK_NK_DOT 47 +#define TK_WITH 48 +#define TK_DNODE 49 +#define TK_PORT 50 +#define TK_DNODES 51 +#define TK_NK_IPTOKEN 52 +#define TK_FORCE 53 +#define TK_LOCAL 54 +#define TK_QNODE 55 +#define TK_BNODE 56 +#define TK_SNODE 57 +#define TK_MNODE 58 +#define TK_DATABASE 59 +#define TK_USE 60 +#define TK_FLUSH 61 +#define TK_TRIM 62 +#define TK_COMPACT 63 +#define TK_IF 64 +#define TK_NOT 65 +#define TK_EXISTS 66 +#define TK_BUFFER 67 +#define TK_CACHEMODEL 68 +#define TK_CACHESIZE 69 +#define TK_COMP 70 +#define TK_DURATION 71 +#define TK_NK_VARIABLE 72 +#define TK_MAXROWS 73 +#define TK_MINROWS 74 +#define TK_KEEP 75 +#define TK_PAGES 76 +#define TK_PAGESIZE 77 +#define TK_TSDB_PAGESIZE 78 +#define TK_PRECISION 79 +#define TK_REPLICA 80 +#define TK_VGROUPS 81 +#define TK_SINGLE_STABLE 82 +#define TK_RETENTIONS 83 +#define TK_SCHEMALESS 84 +#define TK_WAL_LEVEL 85 +#define TK_WAL_FSYNC_PERIOD 86 +#define TK_WAL_RETENTION_PERIOD 87 +#define TK_WAL_RETENTION_SIZE 88 +#define TK_WAL_ROLL_PERIOD 89 +#define TK_WAL_SEGMENT_SIZE 90 +#define TK_STT_TRIGGER 91 +#define TK_TABLE_PREFIX 92 +#define TK_TABLE_SUFFIX 93 +#define TK_NK_COLON 94 +#define TK_MAX_SPEED 95 +#define TK_START 96 +#define TK_TIMESTAMP 97 +#define TK_END 98 +#define TK_TABLE 99 +#define TK_NK_LP 100 +#define TK_NK_RP 101 +#define TK_STABLE 102 +#define TK_ADD 103 +#define TK_COLUMN 104 +#define TK_MODIFY 105 +#define TK_RENAME 106 +#define TK_TAG 107 +#define TK_SET 108 +#define TK_NK_EQ 109 +#define TK_USING 110 +#define TK_TAGS 111 +#define TK_BOOL 112 +#define TK_TINYINT 113 +#define TK_SMALLINT 114 +#define TK_INT 115 +#define TK_INTEGER 116 +#define TK_BIGINT 117 +#define TK_FLOAT 118 +#define TK_DOUBLE 119 +#define TK_BINARY 120 +#define TK_NCHAR 121 +#define TK_UNSIGNED 122 +#define TK_JSON 123 +#define TK_VARCHAR 124 +#define TK_MEDIUMBLOB 125 +#define TK_BLOB 126 +#define TK_VARBINARY 127 +#define TK_DECIMAL 128 +#define TK_COMMENT 129 +#define TK_MAX_DELAY 130 +#define TK_WATERMARK 131 +#define TK_ROLLUP 132 +#define TK_TTL 133 +#define TK_SMA 134 +#define TK_DELETE_MARK 135 +#define TK_FIRST 136 +#define TK_LAST 137 +#define TK_SHOW 138 +#define TK_PRIVILEGES 139 +#define TK_DATABASES 140 +#define TK_TABLES 141 +#define TK_STABLES 142 +#define TK_MNODES 143 +#define TK_QNODES 144 +#define TK_FUNCTIONS 145 +#define TK_INDEXES 146 +#define TK_ACCOUNTS 147 +#define TK_APPS 148 +#define TK_CONNECTIONS 149 +#define TK_LICENCES 150 +#define TK_GRANTS 151 +#define TK_QUERIES 152 +#define TK_SCORES 153 +#define TK_TOPICS 154 +#define TK_VARIABLES 155 +#define TK_CLUSTER 156 +#define TK_BNODES 157 +#define TK_SNODES 158 +#define TK_TRANSACTIONS 159 +#define TK_DISTRIBUTED 160 +#define TK_CONSUMERS 161 +#define TK_SUBSCRIPTIONS 162 +#define TK_VNODES 163 +#define TK_ALIVE 164 +#define TK_LIKE 165 +#define TK_TBNAME 166 +#define TK_QTAGS 167 +#define TK_AS 168 +#define TK_INDEX 169 +#define TK_FUNCTION 170 +#define TK_INTERVAL 171 +#define TK_COUNT 172 +#define TK_LAST_ROW 173 +#define TK_TOPIC 174 +#define TK_META 175 +#define TK_CONSUMER 176 +#define TK_GROUP 177 +#define TK_DESC 178 +#define TK_DESCRIBE 179 +#define TK_RESET 180 +#define TK_QUERY 181 +#define TK_CACHE 182 +#define TK_EXPLAIN 183 +#define TK_ANALYZE 184 +#define TK_VERBOSE 185 +#define TK_NK_BOOL 186 +#define TK_RATIO 187 +#define TK_NK_FLOAT 188 +#define TK_OUTPUTTYPE 189 +#define TK_AGGREGATE 190 +#define TK_BUFSIZE 191 +#define TK_LANGUAGE 192 +#define TK_REPLACE 193 +#define TK_STREAM 194 +#define TK_INTO 195 +#define TK_TRIGGER 196 +#define TK_AT_ONCE 197 +#define TK_WINDOW_CLOSE 198 +#define TK_IGNORE 199 +#define TK_EXPIRED 200 +#define TK_FILL_HISTORY 201 +#define TK_UPDATE 202 +#define TK_SUBTABLE 203 +#define TK_KILL 204 +#define TK_CONNECTION 205 +#define TK_TRANSACTION 206 +#define TK_BALANCE 207 +#define TK_VGROUP 208 +#define TK_LEADER 209 +#define TK_MERGE 210 +#define TK_REDISTRIBUTE 211 +#define TK_SPLIT 212 +#define TK_DELETE 213 +#define TK_INSERT 214 +#define TK_NULL 215 +#define TK_NK_QUESTION 216 +#define TK_NK_ARROW 217 +#define TK_ROWTS 218 +#define TK_QSTART 219 +#define TK_QEND 220 +#define TK_QDURATION 221 +#define TK_WSTART 222 +#define TK_WEND 223 +#define TK_WDURATION 224 +#define TK_IROWTS 225 +#define TK_ISFILLED 226 +#define TK_CAST 227 +#define TK_NOW 228 +#define TK_TODAY 229 +#define TK_TIMEZONE 230 +#define TK_CLIENT_VERSION 231 +#define TK_SERVER_VERSION 232 +#define TK_SERVER_STATUS 233 +#define TK_CURRENT_USER 234 +#define TK_CASE 235 +#define TK_WHEN 236 +#define TK_THEN 237 +#define TK_ELSE 238 +#define TK_BETWEEN 239 +#define TK_IS 240 +#define TK_NK_LT 241 +#define TK_NK_GT 242 +#define TK_NK_LE 243 +#define TK_NK_GE 244 +#define TK_NK_NE 245 +#define TK_MATCH 246 +#define TK_NMATCH 247 +#define TK_CONTAINS 248 +#define TK_IN 249 +#define TK_JOIN 250 +#define TK_INNER 251 +#define TK_SELECT 252 +#define TK_DISTINCT 253 +#define TK_WHERE 254 +#define TK_PARTITION 255 +#define TK_BY 256 +#define TK_SESSION 257 +#define TK_STATE_WINDOW 258 +#define TK_EVENT_WINDOW 259 +#define TK_SLIDING 260 +#define TK_FILL 261 +#define TK_VALUE 262 +#define TK_VALUE_F 263 +#define TK_NONE 264 +#define TK_PREV 265 +#define TK_NULL_F 266 +#define TK_LINEAR 267 +#define TK_NEXT 268 +#define TK_HAVING 269 +#define TK_RANGE 270 +#define TK_EVERY 271 +#define TK_ORDER 272 +#define TK_SLIMIT 273 +#define TK_SOFFSET 274 +#define TK_LIMIT 275 +#define TK_OFFSET 276 +#define TK_ASC 277 +#define TK_NULLS 278 +#define TK_ABORT 279 +#define TK_AFTER 280 +#define TK_ATTACH 281 +#define TK_BEFORE 282 +#define TK_BEGIN 283 +#define TK_BITAND 284 +#define TK_BITNOT 285 +#define TK_BITOR 286 +#define TK_BLOCKS 287 +#define TK_CHANGE 288 +#define TK_COMMA 289 +#define TK_CONCAT 290 +#define TK_CONFLICT 291 +#define TK_COPY 292 +#define TK_DEFERRED 293 +#define TK_DELIMITERS 294 +#define TK_DETACH 295 +#define TK_DIVIDE 296 +#define TK_DOT 297 +#define TK_EACH 298 +#define TK_FAIL 299 +#define TK_FILE 300 +#define TK_FOR 301 +#define TK_GLOB 302 +#define TK_ID 303 +#define TK_IMMEDIATE 304 +#define TK_IMPORT 305 +#define TK_INITIALLY 306 +#define TK_INSTEAD 307 +#define TK_ISNULL 308 +#define TK_KEY 309 +#define TK_MODULES 310 +#define TK_NK_BITNOT 311 +#define TK_NK_SEMI 312 +#define TK_NOTNULL 313 +#define TK_OF 314 +#define TK_PLUS 315 +#define TK_PRIVILEGE 316 +#define TK_RAISE 317 +#define TK_RESTRICT 318 +#define TK_ROW 319 +#define TK_SEMI 320 +#define TK_STAR 321 +#define TK_STATEMENT 322 +#define TK_STRICT 323 +#define TK_STRING 324 +#define TK_TIMES 325 +#define TK_VALUES 326 +#define TK_VARIABLE 327 +#define TK_VIEW 328 +#define TK_WAL 329 + #define TK_NK_SPACE 600 #define TK_NK_COMMENT 601 diff --git a/include/util/tdef.h b/include/util/tdef.h index 2f86395dad..680620a094 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -368,11 +368,11 @@ typedef enum ELogicConditionType { #define TSDB_MIN_STT_TRIGGER 1 #define TSDB_MAX_STT_TRIGGER 16 #define TSDB_DEFAULT_SST_TRIGGER 1 -#define TSDB_MIN_HASH_PREFIX 0 -#define TSDB_MAX_HASH_PREFIX 128 +#define TSDB_MIN_HASH_PREFIX (2 - TSDB_TABLE_NAME_LEN) +#define TSDB_MAX_HASH_PREFIX (TSDB_TABLE_NAME_LEN - 2) #define TSDB_DEFAULT_HASH_PREFIX 0 -#define TSDB_MIN_HASH_SUFFIX 0 -#define TSDB_MAX_HASH_SUFFIX 128 +#define TSDB_MIN_HASH_SUFFIX (2 - TSDB_TABLE_NAME_LEN) +#define TSDB_MAX_HASH_SUFFIX (TSDB_TABLE_NAME_LEN - 2) #define TSDB_DEFAULT_HASH_SUFFIX 0 #define TSDB_DB_MIN_WAL_RETENTION_PERIOD -1 diff --git a/include/util/tutil.h b/include/util/tutil.h index e0801e5295..e96c7a07d9 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -81,14 +81,22 @@ static FORCE_INLINE void taosEncryptPass_c(uint8_t *inBuf, size_t len, char *tar static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, int32_t method, int32_t prefix, int32_t suffix) { - if (prefix == 0 && suffix == 0) { + if ((prefix == 0 && suffix == 0) || (tblen <= (prefix + suffix)) || (tblen <= -1 * (prefix + suffix)) || prefix * suffix < 0) { return MurmurHash3_32(tbname, tblen); + } else if (prefix > 0 || suffix > 0) { + return MurmurHash3_32(tbname + prefix, tblen - prefix - suffix); } else { - if (tblen <= (prefix + suffix)) { - return MurmurHash3_32(tbname, tblen); - } else { - return MurmurHash3_32(tbname + prefix, tblen - prefix - suffix); + char tbName[TSDB_TABLE_FNAME_LEN]; + int32_t offset = 0; + if (prefix < 0) { + offset = -1 * prefix; + strncpy(tbName, tbname, offset); } + if (suffix < 0) { + strncpy(tbName + offset, tbname + tblen + suffix, -1 * suffix); + offset += -1 *suffix; + } + return MurmurHash3_32(tbName, offset); } } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index e0b3e2bf74..23b2b9d7c6 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -373,6 +373,8 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { if (pCfg->sstTrigger < TSDB_MIN_STT_TRIGGER || pCfg->sstTrigger > TSDB_MAX_STT_TRIGGER) return -1; if (pCfg->hashPrefix < TSDB_MIN_HASH_PREFIX || pCfg->hashPrefix > TSDB_MAX_HASH_PREFIX) return -1; if (pCfg->hashSuffix < TSDB_MIN_HASH_SUFFIX || pCfg->hashSuffix > TSDB_MAX_HASH_SUFFIX) return -1; + if ((pCfg->hashSuffix * pCfg->hashPrefix) < 0) return -1; + if ((pCfg->hashPrefix + pCfg->hashSuffix) >= (TSDB_TABLE_NAME_LEN - 1)) return -1; if (pCfg->tsdbPageSize < TSDB_MIN_TSDB_PAGESIZE || pCfg->tsdbPageSize > TSDB_MAX_TSDB_PAGESIZE) return -1; if (taosArrayGetSize(pCfg->pRetensions) != pCfg->numOfRetensions) return -1; @@ -409,8 +411,6 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->walRollPeriod < 0) pCfg->walRollPeriod = TSDB_REPS_DEF_DB_WAL_ROLL_PERIOD; if (pCfg->walSegmentSize < 0) pCfg->walSegmentSize = TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE; if (pCfg->sstTrigger <= 0) pCfg->sstTrigger = TSDB_DEFAULT_SST_TRIGGER; - if (pCfg->hashPrefix < 0) pCfg->hashPrefix = TSDB_DEFAULT_HASH_PREFIX; - if (pCfg->hashSuffix < 0) pCfg->hashSuffix = TSDB_DEFAULT_HASH_SUFFIX; if (pCfg->tsdbPageSize <= 0) pCfg->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE; } @@ -553,6 +553,10 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, int32_t dbLen = strlen(dbObj.name) + 1; mInfo("db:%s, hashPrefix adjust from %d to %d", dbObj.name, dbObj.cfg.hashPrefix, dbObj.cfg.hashPrefix + dbLen); dbObj.cfg.hashPrefix += dbLen; + } else if (dbObj.cfg.hashPrefix < 0) { + int32_t dbLen = strlen(dbObj.name) + 1; + mInfo("db:%s, hashPrefix adjust from %d to %d", dbObj.name, dbObj.cfg.hashPrefix, dbObj.cfg.hashPrefix - dbLen); + dbObj.cfg.hashPrefix -= dbLen; } SVgObj *pVgroups = NULL; @@ -1788,6 +1792,8 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, int16_t hashPrefix = pDb->cfg.hashPrefix; if (hashPrefix > 0) { hashPrefix = pDb->cfg.hashPrefix - strlen(pDb->name) - 1; + } else if (hashPrefix < 0) { + hashPrefix = pDb->cfg.hashPrefix + strlen(pDb->name) + 1; } colDataSetVal(pColInfo, rows, (const char *)&hashPrefix, false); diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 58c43829cf..6efdd8d8eb 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -278,7 +278,12 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, ch char* retentions = buildRetension(pCfg->pRetensions); int32_t dbFNameLen = strlen(dbFName); - int32_t hashPrefix = (pCfg->hashPrefix > (dbFNameLen + 1)) ? (pCfg->hashPrefix - dbFNameLen - 1) : 0; + int32_t hashPrefix = 0; + if (pCfg->hashPrefix > 0) { + hashPrefix = pCfg->hashPrefix - dbFNameLen - 1; + } else if (pCfg->hashPrefix < 0) { + hashPrefix = pCfg->hashPrefix + dbFNameLen + 1; + } len += sprintf( buf2 + VARSTR_HEADER_SIZE, diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y old mode 100644 new mode 100755 index f99976e0df..5918b488b4 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -221,8 +221,8 @@ db_options(A) ::= db_options(B) WAL_RETENTION_SIZE NK_MINUS(D) NK_INTEGER(C). db_options(A) ::= db_options(B) WAL_ROLL_PERIOD NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_WAL_ROLL_PERIOD, &C); } db_options(A) ::= db_options(B) WAL_SEGMENT_SIZE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_WAL_SEGMENT_SIZE, &C); } db_options(A) ::= db_options(B) STT_TRIGGER NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STT_TRIGGER, &C); } -db_options(A) ::= db_options(B) TABLE_PREFIX NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_PREFIX, &C); } -db_options(A) ::= db_options(B) TABLE_SUFFIX NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_SUFFIX, &C); } +db_options(A) ::= db_options(B) TABLE_PREFIX signed(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_PREFIX, C); } +db_options(A) ::= db_options(B) TABLE_SUFFIX signed(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_SUFFIX, C); } alter_db_options(A) ::= alter_db_option(B). { A = createAlterDatabaseOptions(pCxt); A = setAlterDatabaseOption(pCxt, A, &B); } alter_db_options(A) ::= alter_db_options(B) alter_db_option(C). { A = setAlterDatabaseOption(pCxt, B, &C); } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 2afe34c1f7..0ba57af29c 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -1024,12 +1024,26 @@ static SNode* setDatabaseOptionImpl(SAstCreateContext* pCxt, SNode* pOptions, ED case DB_OPTION_STT_TRIGGER: pDbOptions->sstTrigger = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; - case DB_OPTION_TABLE_PREFIX: - pDbOptions->tablePrefix = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); + case DB_OPTION_TABLE_PREFIX: { + SValueNode *pNode = (SValueNode *)pVal; + if (TSDB_DATA_TYPE_BIGINT == pNode->node.resType.type || TSDB_DATA_TYPE_UBIGINT == pNode->node.resType.type) { + pDbOptions->tablePrefix = taosStr2Int32(pNode->literal, NULL, 10); + } else { + snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid table_prefix data type"); + pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; + } break; - case DB_OPTION_TABLE_SUFFIX: - pDbOptions->tableSuffix = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); + } + case DB_OPTION_TABLE_SUFFIX:{ + SValueNode *pNode = (SValueNode *)pVal; + if (TSDB_DATA_TYPE_BIGINT == pNode->node.resType.type || TSDB_DATA_TYPE_UBIGINT == pNode->node.resType.type) { + pDbOptions->tableSuffix = taosStr2Int32(pNode->literal, NULL, 10); + } else { + snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid table_suffix data type"); + pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; + } break; + } default: break; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index b44c36dde1..c0ee34d6aa 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4177,6 +4177,34 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete return TSDB_CODE_SUCCESS; } + +static int32_t checkDbTbPrefixSuffixOptions(STranslateContext* pCxt, int32_t tbPrefix, int32_t tbSuffix) { + if (tbPrefix < TSDB_MIN_HASH_PREFIX || tbPrefix > TSDB_MAX_HASH_PREFIX) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid option table_prefix: %d valid range: [%" PRId64 ", %" PRId64 "]", tbPrefix, + TSDB_MIN_HASH_PREFIX, TSDB_MAX_HASH_PREFIX); + } + + if (tbSuffix < TSDB_MIN_HASH_SUFFIX || tbSuffix > TSDB_MAX_HASH_SUFFIX) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid option table_suffix: %d valid range: [%" PRId64 ", %" PRId64 "]", tbSuffix, + TSDB_MIN_HASH_SUFFIX, TSDB_MAX_HASH_SUFFIX); + } + + if ((tbPrefix * tbSuffix) < 0) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid option table_prefix & table_suffix: mixed usage not allowed"); + } + + if ((tbPrefix + tbSuffix) >= (TSDB_TABLE_NAME_LEN - 1)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid option table_prefix & table_suffix: exceed max table name length"); + } + + return TSDB_CODE_SUCCESS; +} + + static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions) { int32_t daysPerFile = pOptions->daysPerFile; int64_t daysToKeep0 = pOptions->keep[0]; @@ -4284,10 +4312,7 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName code = checkDbRangeOption(pCxt, "sstTrigger", pOptions->sstTrigger, TSDB_MIN_STT_TRIGGER, TSDB_MAX_STT_TRIGGER); } if (TSDB_CODE_SUCCESS == code) { - code = checkDbRangeOption(pCxt, "tablePrefix", pOptions->tablePrefix, TSDB_MIN_HASH_PREFIX, TSDB_MAX_HASH_PREFIX); - } - if (TSDB_CODE_SUCCESS == code) { - code = checkDbRangeOption(pCxt, "tableSuffix", pOptions->tableSuffix, TSDB_MIN_HASH_SUFFIX, TSDB_MAX_HASH_SUFFIX); + code = checkDbTbPrefixSuffixOptions(pCxt, pOptions->tablePrefix, pOptions->tableSuffix); } if (TSDB_CODE_SUCCESS == code) { code = checkOptionsDependency(pCxt, pDbName, pOptions); diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 9ad5bcf644..2e473455e4 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -142,6 +142,7 @@ typedef union { #define YYFALLBACK 1 #define YYNSTATE 762 #define YYNRULE 583 +#define YYNRULE_WITH_ACTION 583 #define YYNTOKEN 330 #define YY_MAX_SHIFT 761 #define YY_MIN_SHIFTREDUCE 1136 @@ -217,711 +218,744 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (2905) +#define YY_ACTTAB_COUNT (2904) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 2116, 1881, 2010, 499, 432, 1877, 500, 1758, 431, 2102, - /* 10 */ 670, 2051, 46, 44, 1646, 1722, 2102, 2008, 640, 2098, - /* 20 */ 393, 504, 1495, 1520, 39, 38, 2098, 501, 45, 43, - /* 30 */ 42, 41, 40, 1576, 1791, 1493, 2134, 2010, 1520, 132, - /* 40 */ 131, 130, 129, 128, 127, 126, 125, 124, 2084, 384, - /* 50 */ 669, 591, 2007, 640, 2275, 2094, 2100, 374, 244, 1571, - /* 60 */ 28, 1944, 2094, 2100, 375, 19, 663, 652, 371, 2281, - /* 70 */ 184, 638, 1501, 663, 2276, 617, 1942, 628, 140, 1869, - /* 80 */ 507, 2115, 107, 500, 1758, 2151, 36, 296, 169, 2117, - /* 90 */ 673, 2119, 2120, 668, 168, 663, 1734, 758, 141, 9, - /* 100 */ 15, 735, 734, 733, 732, 403, 1884, 731, 730, 144, + /* 0 */ 2116, 2010, 1881, 396, 434, 168, 636, 1734, 433, 1989, + /* 10 */ 672, 162, 46, 44, 1646, 1723, 2008, 642, 374, 1894, + /* 20 */ 393, 506, 1495, 440, 39, 38, 1942, 503, 45, 43, + /* 30 */ 42, 41, 40, 1576, 1791, 1493, 123, 2134, 1520, 122, + /* 40 */ 121, 120, 119, 118, 117, 116, 115, 114, 2279, 2084, + /* 50 */ 593, 671, 593, 2275, 194, 2275, 39, 38, 167, 1571, + /* 60 */ 45, 43, 42, 41, 40, 19, 1833, 340, 2281, 186, + /* 70 */ 2281, 186, 1501, 2276, 619, 2276, 619, 45, 43, 42, + /* 80 */ 41, 40, 2115, 501, 654, 2151, 502, 1758, 169, 2117, + /* 90 */ 675, 2119, 2120, 670, 182, 665, 1520, 758, 36, 298, + /* 100 */ 15, 735, 734, 733, 732, 405, 1931, 731, 730, 144, /* 110 */ 725, 724, 723, 722, 721, 720, 719, 157, 715, 714, - /* 120 */ 713, 402, 401, 710, 709, 708, 707, 706, 592, 2241, - /* 130 */ 398, 2219, 1320, 1937, 1939, 123, 1578, 1579, 122, 121, - /* 140 */ 120, 119, 118, 117, 116, 115, 114, 1311, 695, 694, - /* 150 */ 693, 1315, 692, 1317, 1318, 691, 688, 2216, 1326, 685, - /* 160 */ 1328, 1329, 682, 679, 177, 652, 1551, 1561, 2280, 1409, - /* 170 */ 1410, 2275, 1577, 1580, 1944, 653, 1892, 630, 182, 2212, - /* 180 */ 2213, 356, 138, 2217, 358, 1993, 1496, 2279, 1494, 1942, - /* 190 */ 1720, 2276, 2278, 133, 287, 288, 516, 39, 38, 286, - /* 200 */ 537, 45, 43, 42, 41, 40, 278, 62, 703, 155, - /* 210 */ 154, 700, 699, 698, 152, 1499, 1500, 1794, 1550, 1553, - /* 220 */ 1554, 1555, 1556, 1557, 1558, 1559, 1560, 665, 661, 1569, - /* 230 */ 1570, 1572, 1573, 1574, 1575, 2, 46, 44, 425, 1169, - /* 240 */ 1522, 341, 62, 1518, 393, 49, 1495, 62, 611, 93, - /* 250 */ 469, 2116, 616, 483, 350, 2275, 482, 1576, 177, 1493, - /* 260 */ 406, 670, 427, 423, 405, 45, 43, 42, 41, 40, - /* 270 */ 615, 184, 452, 50, 484, 2276, 617, 454, 1171, 1994, - /* 280 */ 1174, 1175, 180, 1571, 555, 554, 553, 2134, 1723, 19, - /* 290 */ 106, 545, 137, 549, 1931, 1520, 1501, 548, 1605, 2084, - /* 300 */ 103, 669, 547, 552, 366, 365, 1521, 591, 546, 123, - /* 310 */ 2275, 1639, 122, 121, 120, 119, 118, 117, 116, 115, - /* 320 */ 114, 758, 359, 101, 15, 2281, 184, 430, 2280, 429, - /* 330 */ 2276, 617, 2115, 1191, 442, 1190, 2151, 653, 1892, 110, - /* 340 */ 2117, 673, 2119, 2120, 668, 1522, 663, 1885, 226, 143, - /* 350 */ 438, 150, 2175, 2204, 1606, 133, 428, 387, 2200, 187, - /* 360 */ 1578, 1579, 542, 480, 1519, 1192, 474, 473, 472, 471, - /* 370 */ 468, 467, 466, 465, 464, 460, 459, 458, 457, 340, - /* 380 */ 449, 448, 447, 209, 444, 443, 357, 502, 277, 1765, - /* 390 */ 1551, 1561, 2280, 338, 187, 2275, 1577, 1580, 1650, 187, - /* 400 */ 555, 554, 553, 705, 1520, 1938, 1939, 545, 137, 549, - /* 410 */ 1496, 2279, 1494, 548, 606, 2276, 2277, 1868, 547, 552, - /* 420 */ 366, 365, 1354, 1355, 546, 187, 1708, 1264, 35, 391, + /* 120 */ 713, 404, 403, 710, 709, 708, 175, 174, 594, 2241, + /* 130 */ 509, 654, 1320, 502, 1758, 123, 1578, 1579, 122, 121, + /* 140 */ 120, 119, 118, 117, 116, 115, 114, 1311, 697, 696, + /* 150 */ 695, 1315, 694, 1317, 1318, 693, 690, 179, 1326, 687, + /* 160 */ 1328, 1329, 684, 681, 49, 640, 1551, 1561, 654, 2219, + /* 170 */ 655, 1892, 1577, 1580, 1720, 39, 38, 360, 1993, 45, + /* 180 */ 43, 42, 41, 40, 1409, 1410, 1496, 1523, 1494, 133, + /* 190 */ 655, 1892, 387, 62, 1715, 2216, 539, 39, 38, 280, + /* 200 */ 165, 45, 43, 42, 41, 40, 39, 38, 1894, 191, + /* 210 */ 45, 43, 42, 41, 40, 1499, 1500, 1868, 1550, 1553, + /* 220 */ 1554, 1555, 1556, 1557, 1558, 1559, 1560, 667, 663, 1569, + /* 230 */ 1570, 1572, 1573, 1574, 1575, 2, 46, 44, 1722, 179, + /* 240 */ 62, 343, 93, 1518, 393, 408, 1495, 62, 49, 407, + /* 250 */ 471, 2116, 2280, 485, 352, 2275, 484, 1576, 1989, 1493, + /* 260 */ 1994, 672, 132, 131, 130, 129, 128, 127, 126, 125, + /* 270 */ 124, 2279, 454, 1522, 486, 2276, 2278, 456, 432, 396, + /* 280 */ 431, 707, 2280, 1571, 1275, 2275, 211, 165, 2134, 19, + /* 290 */ 504, 593, 1765, 1714, 2275, 1894, 1501, 1274, 1605, 1685, + /* 300 */ 2084, 2279, 671, 196, 2010, 2276, 2277, 430, 618, 2281, + /* 310 */ 186, 2275, 518, 2116, 2276, 619, 386, 621, 66, 2007, + /* 320 */ 642, 758, 361, 672, 15, 1767, 617, 186, 1264, 655, + /* 330 */ 1892, 2276, 619, 2115, 444, 257, 2151, 655, 1892, 110, + /* 340 */ 2117, 675, 2119, 2120, 670, 189, 665, 1521, 133, 143, + /* 350 */ 2134, 150, 2175, 2204, 1606, 544, 55, 389, 2200, 487, + /* 360 */ 1578, 1579, 2084, 482, 671, 1266, 476, 475, 474, 473, + /* 370 */ 470, 469, 468, 467, 466, 462, 461, 460, 459, 342, + /* 380 */ 451, 450, 449, 613, 446, 445, 359, 1650, 1520, 1521, + /* 390 */ 1551, 1561, 189, 1520, 545, 2115, 1577, 1580, 2151, 189, + /* 400 */ 189, 110, 2117, 675, 2119, 2120, 670, 707, 665, 1191, + /* 410 */ 1496, 1190, 1494, 2295, 62, 2204, 1262, 630, 140, 389, + /* 420 */ 2200, 279, 50, 608, 641, 630, 140, 1745, 35, 391, /* 430 */ 1600, 1601, 1602, 1603, 1604, 1608, 1609, 1610, 1611, 1499, - /* 440 */ 1500, 1552, 1550, 1553, 1554, 1555, 1556, 1557, 1558, 1559, - /* 450 */ 1560, 665, 661, 1569, 1570, 1572, 1573, 1574, 1575, 2, - /* 460 */ 12, 46, 44, 227, 1266, 1495, 167, 1521, 211, 393, - /* 470 */ 2116, 1495, 502, 1833, 1765, 1674, 578, 652, 1493, 172, - /* 480 */ 631, 705, 1576, 476, 1493, 533, 529, 525, 521, 224, - /* 490 */ 2219, 39, 38, 277, 1745, 45, 43, 42, 41, 40, - /* 500 */ 612, 607, 600, 1523, 1523, 66, 2134, 1883, 1571, 551, - /* 510 */ 550, 1501, 1520, 610, 19, 1501, 2215, 2098, 2084, 639, - /* 520 */ 669, 1501, 603, 602, 1672, 1673, 1675, 1676, 1677, 88, - /* 530 */ 39, 38, 222, 12, 45, 43, 42, 41, 40, 2134, - /* 540 */ 758, 1944, 2084, 200, 199, 543, 758, 1744, 381, 15, - /* 550 */ 1552, 2115, 1607, 2094, 2100, 2151, 1942, 49, 110, 2117, - /* 560 */ 673, 2119, 2120, 668, 663, 663, 475, 1262, 166, 514, - /* 570 */ 181, 2003, 2204, 316, 39, 38, 387, 2200, 45, 43, - /* 580 */ 42, 41, 40, 1426, 1427, 1578, 1579, 314, 73, 186, - /* 590 */ 1743, 72, 62, 609, 560, 2084, 1191, 2230, 1190, 221, - /* 600 */ 215, 62, 87, 639, 220, 12, 512, 10, 2219, 570, - /* 610 */ 207, 495, 493, 490, 696, 1551, 1561, 396, 360, 1425, - /* 620 */ 1428, 1577, 1580, 240, 213, 162, 33, 1887, 1192, 1496, - /* 630 */ 717, 1494, 372, 1894, 2214, 1496, 1612, 1494, 2084, 563, - /* 640 */ 1942, 2067, 653, 1892, 557, 653, 1892, 1944, 1742, 239, - /* 650 */ 62, 255, 193, 637, 386, 2003, 2102, 541, 1499, 1500, - /* 660 */ 189, 540, 1942, 55, 1499, 1500, 2098, 1550, 1553, 1554, - /* 670 */ 1555, 1556, 1557, 1558, 1559, 1560, 665, 661, 1569, 1570, + /* 440 */ 1500, 1192, 1550, 1553, 1554, 1555, 1556, 1557, 1558, 1559, + /* 450 */ 1560, 667, 663, 1569, 1570, 1572, 1573, 1574, 1575, 2, + /* 460 */ 12, 46, 44, 1354, 1355, 1495, 402, 401, 62, 393, + /* 470 */ 1279, 1495, 557, 556, 555, 516, 2084, 2003, 1493, 547, + /* 480 */ 137, 551, 1576, 1278, 1493, 550, 2280, 2116, 32, 1502, + /* 490 */ 549, 554, 368, 367, 39, 38, 548, 672, 45, 43, + /* 500 */ 42, 41, 40, 1522, 1523, 1426, 1427, 2116, 1571, 614, + /* 510 */ 609, 602, 1674, 699, 19, 1501, 1935, 669, 185, 2212, + /* 520 */ 2213, 1501, 138, 2217, 2134, 632, 184, 2212, 2213, 398, + /* 530 */ 138, 2217, 1937, 1939, 2051, 1191, 2084, 1190, 671, 1552, + /* 540 */ 758, 1425, 1428, 1519, 2134, 489, 758, 39, 38, 15, + /* 550 */ 228, 45, 43, 42, 41, 40, 2084, 153, 671, 605, + /* 560 */ 604, 1672, 1673, 1675, 1676, 1677, 189, 1192, 166, 2115, + /* 570 */ 1744, 580, 2151, 318, 189, 170, 2117, 675, 2119, 2120, + /* 580 */ 670, 246, 665, 28, 1708, 1578, 1579, 316, 73, 2115, + /* 590 */ 12, 72, 2151, 2094, 562, 334, 2117, 675, 2119, 2120, + /* 600 */ 670, 668, 665, 656, 2169, 655, 1892, 2102, 107, 572, + /* 610 */ 209, 497, 495, 492, 54, 1551, 1561, 2098, 612, 2084, + /* 620 */ 189, 1577, 1580, 242, 438, 141, 620, 2296, 213, 1496, + /* 630 */ 478, 1494, 504, 1884, 1765, 1496, 1794, 1494, 165, 565, + /* 640 */ 1944, 1944, 1505, 641, 559, 2134, 1895, 358, 373, 241, + /* 650 */ 62, 279, 195, 2100, 376, 1942, 1942, 1501, 1499, 1500, + /* 660 */ 1169, 289, 290, 665, 1499, 1500, 288, 1550, 1553, 1554, + /* 670 */ 1555, 1556, 1557, 1558, 1559, 1560, 667, 663, 1569, 1570, /* 680 */ 1572, 1573, 1574, 1575, 2, 46, 44, 1581, 109, 70, - /* 690 */ 1944, 2116, 69, 393, 1879, 1495, 2084, 397, 653, 1892, - /* 700 */ 1715, 631, 2094, 2100, 388, 1942, 1576, 1741, 1493, 187, - /* 710 */ 718, 32, 1854, 663, 628, 140, 436, 39, 38, 653, - /* 720 */ 1892, 45, 43, 42, 41, 40, 1834, 2134, 81, 80, - /* 730 */ 435, 87, 1571, 191, 164, 729, 727, 437, 639, 2084, - /* 740 */ 1875, 669, 39, 38, 187, 1501, 45, 43, 42, 41, - /* 750 */ 40, 653, 1892, 187, 339, 2084, 1888, 421, 1974, 1989, - /* 760 */ 419, 415, 411, 408, 428, 1177, 1740, 653, 1892, 446, - /* 770 */ 758, 1519, 2115, 47, 653, 1892, 2151, 2116, 1739, 110, - /* 780 */ 2117, 673, 2119, 2120, 668, 461, 663, 670, 648, 1767, - /* 790 */ 2003, 181, 462, 2204, 34, 1643, 1738, 387, 2200, 1714, - /* 800 */ 39, 38, 187, 192, 45, 43, 42, 41, 40, 1578, - /* 810 */ 1579, 653, 1892, 2134, 2084, 275, 2212, 627, 2231, 134, - /* 820 */ 626, 569, 2275, 628, 140, 2084, 2084, 669, 1896, 515, - /* 830 */ 1523, 653, 1892, 196, 567, 1685, 565, 615, 184, 1551, - /* 840 */ 1561, 1989, 2276, 617, 2084, 1577, 1580, 39, 38, 1889, - /* 850 */ 420, 45, 43, 42, 41, 40, 1867, 142, 2115, 1496, - /* 860 */ 2175, 1494, 2151, 653, 1892, 110, 2117, 673, 2119, 2120, - /* 870 */ 668, 1944, 663, 84, 242, 249, 83, 2295, 241, 2204, - /* 880 */ 363, 245, 165, 387, 2200, 194, 1943, 634, 1499, 1500, - /* 890 */ 1895, 1550, 1553, 1554, 1555, 1556, 1557, 1558, 1559, 1560, - /* 900 */ 665, 661, 1569, 1570, 1572, 1573, 1574, 1575, 2, 46, - /* 910 */ 44, 2116, 655, 456, 2176, 653, 1892, 393, 1737, 1495, - /* 920 */ 1736, 670, 455, 2238, 183, 2212, 2213, 253, 138, 2217, - /* 930 */ 1576, 2116, 1493, 587, 591, 1174, 1175, 2275, 90, 345, - /* 940 */ 1662, 667, 370, 657, 571, 2176, 364, 2134, 362, 361, - /* 950 */ 1733, 539, 2281, 184, 1586, 1735, 1571, 2276, 617, 2084, - /* 960 */ 1520, 669, 653, 1892, 653, 1892, 2084, 2134, 2084, 1501, - /* 970 */ 91, 1275, 541, 42, 41, 40, 540, 2279, 254, 2084, - /* 980 */ 632, 669, 636, 2103, 1274, 703, 155, 154, 700, 699, - /* 990 */ 698, 152, 2115, 2098, 758, 2077, 2151, 47, 2084, 110, - /* 1000 */ 2117, 673, 2119, 2120, 668, 2116, 663, 2078, 14, 13, - /* 1010 */ 1732, 2295, 2115, 2204, 1731, 670, 2151, 387, 2200, 332, - /* 1020 */ 2117, 673, 2119, 2120, 668, 666, 663, 654, 2169, 2094, - /* 1030 */ 2100, 39, 38, 1578, 1579, 45, 43, 42, 41, 40, - /* 1040 */ 663, 2134, 591, 385, 1642, 2275, 485, 1989, 620, 653, - /* 1050 */ 1892, 165, 660, 2084, 591, 669, 664, 2275, 2084, 1894, - /* 1060 */ 2281, 184, 2084, 1551, 1561, 2276, 617, 291, 697, 1577, - /* 1070 */ 1580, 1935, 2281, 184, 2224, 1639, 396, 2276, 617, 1717, - /* 1080 */ 1718, 653, 1892, 1496, 165, 1494, 2115, 653, 1892, 1279, - /* 1090 */ 2151, 198, 1894, 170, 2117, 673, 2119, 2120, 668, 650, - /* 1100 */ 663, 576, 1278, 628, 140, 651, 1552, 52, 1619, 3, - /* 1110 */ 243, 701, 1499, 1500, 1935, 1550, 1553, 1554, 1555, 1556, - /* 1120 */ 1557, 1558, 1559, 1560, 665, 661, 1569, 1570, 1572, 1573, - /* 1130 */ 1574, 1575, 2, 46, 44, 653, 1892, 653, 1892, 1730, - /* 1140 */ 1729, 393, 399, 1495, 618, 2296, 1728, 2116, 591, 2070, - /* 1150 */ 165, 2275, 1727, 297, 1576, 400, 1493, 670, 1894, 2251, - /* 1160 */ 153, 623, 439, 1726, 487, 1725, 2281, 184, 1870, 252, - /* 1170 */ 310, 2276, 617, 1921, 2116, 440, 702, 1468, 1469, 1935, - /* 1180 */ 1571, 74, 232, 2134, 670, 230, 598, 2084, 2084, 146, - /* 1190 */ 573, 135, 572, 1501, 2084, 2084, 616, 669, 413, 2275, - /* 1200 */ 2084, 590, 1597, 544, 185, 2212, 2213, 1781, 138, 2217, - /* 1210 */ 2134, 2084, 148, 2084, 615, 184, 2244, 54, 758, 2276, - /* 1220 */ 617, 15, 2084, 234, 669, 1260, 233, 272, 2115, 556, - /* 1230 */ 82, 1504, 2151, 153, 2116, 110, 2117, 673, 2119, 2120, - /* 1240 */ 668, 1503, 663, 604, 670, 236, 619, 2295, 235, 2204, - /* 1250 */ 1774, 1772, 225, 387, 2200, 2115, 153, 1578, 1579, 2151, - /* 1260 */ 64, 711, 110, 2117, 673, 2119, 2120, 668, 238, 663, - /* 1270 */ 2134, 237, 558, 561, 2295, 64, 2204, 259, 621, 266, - /* 1280 */ 387, 2200, 2084, 1240, 669, 2105, 2135, 1551, 1561, 1998, - /* 1290 */ 1463, 1221, 404, 1577, 1580, 1759, 703, 155, 154, 700, - /* 1300 */ 699, 698, 152, 14, 13, 153, 1764, 1496, 48, 1494, - /* 1310 */ 1932, 284, 2234, 1466, 71, 2115, 151, 1671, 153, 2151, - /* 1320 */ 629, 53, 169, 2117, 673, 2119, 2120, 668, 1222, 663, - /* 1330 */ 48, 1768, 1670, 64, 261, 48, 1499, 1500, 2107, 1550, - /* 1340 */ 1553, 1554, 1555, 1556, 1557, 1558, 1559, 1560, 665, 661, - /* 1350 */ 1569, 1570, 1572, 1573, 1574, 1575, 2, 271, 274, 390, - /* 1360 */ 389, 677, 635, 2242, 151, 1423, 153, 2116, 289, 1509, - /* 1370 */ 136, 645, 151, 293, 712, 1305, 1, 670, 5, 2269, - /* 1380 */ 1576, 753, 1502, 407, 1507, 412, 354, 309, 1446, 441, - /* 1390 */ 1613, 304, 1562, 624, 1506, 197, 1238, 2116, 1523, 1999, - /* 1400 */ 445, 478, 450, 2134, 1518, 463, 1571, 670, 1991, 2223, - /* 1410 */ 470, 477, 479, 488, 489, 2084, 202, 669, 1332, 1501, - /* 1420 */ 486, 1336, 201, 1343, 491, 492, 204, 1341, 494, 156, - /* 1430 */ 496, 1524, 497, 2134, 4, 498, 505, 506, 1526, 508, - /* 1440 */ 212, 1521, 509, 214, 659, 2084, 1525, 669, 2115, 510, - /* 1450 */ 1527, 511, 2151, 217, 513, 110, 2117, 673, 2119, 2120, - /* 1460 */ 668, 219, 663, 85, 86, 2116, 1194, 2295, 517, 2204, - /* 1470 */ 223, 536, 534, 387, 2200, 670, 535, 538, 2115, 344, - /* 1480 */ 2060, 1882, 2151, 2057, 229, 110, 2117, 673, 2119, 2120, - /* 1490 */ 668, 1878, 663, 112, 577, 575, 231, 2295, 89, 2204, - /* 1500 */ 158, 2134, 159, 387, 2200, 149, 1880, 1876, 160, 2056, - /* 1510 */ 246, 161, 582, 2084, 580, 669, 305, 581, 585, 250, - /* 1520 */ 1453, 588, 8, 605, 2250, 248, 643, 595, 2249, 586, - /* 1530 */ 257, 601, 614, 1510, 2235, 1505, 2245, 376, 608, 2226, - /* 1540 */ 265, 173, 596, 594, 260, 268, 2115, 267, 593, 622, - /* 1550 */ 2151, 2298, 625, 110, 2117, 673, 2119, 2120, 668, 269, - /* 1560 */ 663, 377, 1513, 1515, 2274, 2179, 139, 2204, 270, 1639, - /* 1570 */ 1522, 387, 2200, 2116, 2220, 661, 1569, 1570, 1572, 1573, - /* 1580 */ 1574, 1575, 279, 670, 633, 380, 1528, 96, 2004, 306, - /* 1590 */ 641, 642, 2018, 2116, 2017, 2016, 307, 646, 273, 383, - /* 1600 */ 98, 647, 308, 670, 61, 100, 1893, 2185, 102, 2134, - /* 1610 */ 1936, 311, 754, 1855, 2076, 755, 675, 757, 51, 346, - /* 1620 */ 2075, 2084, 347, 669, 2074, 315, 300, 335, 320, 2134, - /* 1630 */ 313, 334, 78, 2071, 409, 1486, 410, 1487, 324, 190, - /* 1640 */ 414, 2084, 416, 669, 2069, 417, 418, 2068, 355, 2066, - /* 1650 */ 422, 2065, 424, 2064, 2115, 79, 426, 1449, 2151, 1448, - /* 1660 */ 2030, 110, 2117, 673, 2119, 2120, 668, 2029, 663, 2028, - /* 1670 */ 433, 434, 2027, 2177, 2115, 2204, 1400, 2116, 2151, 387, - /* 1680 */ 2200, 110, 2117, 673, 2119, 2120, 668, 670, 663, 2026, - /* 1690 */ 1982, 1981, 1979, 656, 145, 2204, 1978, 1977, 1980, 387, - /* 1700 */ 2200, 2116, 1976, 1975, 1973, 1972, 1971, 195, 451, 1970, - /* 1710 */ 453, 670, 1984, 2134, 1969, 1968, 1967, 1966, 1965, 1964, - /* 1720 */ 1963, 1962, 1961, 1960, 1959, 2084, 1958, 669, 1957, 1956, - /* 1730 */ 1955, 1954, 1953, 1952, 1983, 147, 1951, 2134, 1950, 1949, - /* 1740 */ 1948, 1947, 481, 1946, 1945, 1797, 203, 1402, 342, 2084, - /* 1750 */ 1796, 669, 1795, 205, 206, 343, 1793, 1276, 2115, 1280, - /* 1760 */ 1754, 1176, 2151, 218, 2024, 111, 2117, 673, 2119, 2120, - /* 1770 */ 668, 178, 663, 1753, 2047, 2037, 2025, 1272, 2116, 2204, - /* 1780 */ 2002, 1871, 2115, 2203, 2200, 76, 2151, 77, 670, 111, - /* 1790 */ 2117, 673, 2119, 2120, 668, 208, 663, 216, 2104, 210, - /* 1800 */ 1792, 2116, 179, 2204, 503, 1790, 518, 658, 2200, 520, - /* 1810 */ 519, 670, 1788, 522, 2134, 1214, 523, 1786, 524, 526, - /* 1820 */ 1784, 527, 528, 530, 2116, 532, 2084, 1771, 669, 1770, - /* 1830 */ 1750, 531, 1873, 1348, 670, 1347, 1872, 2134, 1263, 1261, - /* 1840 */ 726, 1259, 1258, 1257, 1256, 1255, 728, 2116, 1250, 2084, - /* 1850 */ 1252, 669, 1782, 1251, 1249, 367, 1775, 670, 1773, 671, - /* 1860 */ 2134, 228, 368, 2151, 369, 559, 111, 2117, 673, 2119, - /* 1870 */ 2120, 668, 2084, 663, 669, 63, 562, 1749, 564, 1748, - /* 1880 */ 2204, 566, 2115, 2134, 349, 2200, 2151, 1747, 568, 111, - /* 1890 */ 2117, 673, 2119, 2120, 668, 2084, 663, 669, 113, 1473, - /* 1900 */ 1475, 27, 1472, 2204, 2046, 2115, 1459, 1455, 2201, 2151, - /* 1910 */ 2116, 67, 326, 2117, 673, 2119, 2120, 668, 1457, 663, - /* 1920 */ 670, 2036, 1477, 583, 2023, 2021, 2280, 20, 2115, 1687, - /* 1930 */ 56, 17, 2151, 6, 29, 170, 2117, 673, 2119, 2120, - /* 1940 */ 668, 7, 663, 589, 256, 584, 2134, 597, 258, 599, - /* 1950 */ 59, 382, 60, 373, 163, 613, 1669, 171, 2084, 251, - /* 1960 */ 669, 262, 30, 263, 1661, 264, 21, 65, 92, 2105, - /* 1970 */ 31, 1707, 1708, 22, 1702, 2116, 1701, 378, 1706, 1705, - /* 1980 */ 379, 1636, 1635, 2022, 276, 667, 2020, 2297, 2019, 2001, - /* 1990 */ 58, 2115, 94, 95, 174, 2151, 2116, 282, 333, 2117, - /* 2000 */ 673, 2119, 2120, 668, 283, 663, 670, 23, 644, 2116, - /* 2010 */ 1667, 2134, 285, 290, 68, 2000, 97, 292, 295, 670, - /* 2020 */ 103, 13, 24, 2084, 1511, 669, 1588, 99, 1587, 11, - /* 2030 */ 1543, 1598, 2134, 2154, 175, 1566, 1564, 392, 662, 188, - /* 2040 */ 57, 1563, 672, 674, 2084, 2134, 669, 18, 37, 16, - /* 2050 */ 394, 25, 676, 1535, 1333, 26, 2115, 2084, 395, 669, - /* 2060 */ 2151, 579, 678, 332, 2117, 673, 2119, 2120, 668, 1330, - /* 2070 */ 663, 680, 2170, 681, 683, 1327, 684, 2115, 686, 761, - /* 2080 */ 1321, 2151, 687, 689, 333, 2117, 673, 2119, 2120, 668, - /* 2090 */ 2115, 663, 1319, 303, 2151, 690, 2116, 333, 2117, 673, - /* 2100 */ 2119, 2120, 668, 1325, 663, 104, 670, 1324, 298, 176, - /* 2110 */ 105, 1342, 1323, 75, 1338, 751, 747, 743, 739, 301, - /* 2120 */ 2116, 1322, 1212, 704, 1246, 1245, 1244, 1270, 1243, 1242, - /* 2130 */ 670, 1241, 2134, 1239, 1237, 1236, 1235, 1233, 716, 299, - /* 2140 */ 1232, 1231, 1230, 1229, 2084, 1228, 669, 1227, 1265, 1267, - /* 2150 */ 1224, 1223, 1218, 1220, 1219, 1789, 2134, 1217, 736, 108, - /* 2160 */ 737, 1787, 294, 738, 740, 742, 1785, 744, 2084, 1783, - /* 2170 */ 669, 748, 1769, 746, 741, 752, 745, 574, 750, 1166, - /* 2180 */ 749, 2151, 1746, 2116, 328, 2117, 673, 2119, 2120, 668, - /* 2190 */ 302, 663, 756, 670, 649, 1497, 312, 759, 760, 1721, - /* 2200 */ 1721, 2115, 1721, 1721, 1721, 2151, 1721, 2116, 317, 2117, - /* 2210 */ 673, 2119, 2120, 668, 1721, 663, 1721, 670, 1721, 2134, - /* 2220 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 281, - /* 2230 */ 1721, 2084, 1721, 669, 280, 1721, 1721, 1721, 1721, 1721, - /* 2240 */ 1721, 1721, 1721, 2134, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2250 */ 1721, 1721, 1721, 2116, 247, 2084, 1721, 669, 1721, 1721, - /* 2260 */ 1721, 1721, 1721, 670, 2115, 1721, 1721, 1721, 2151, 2116, - /* 2270 */ 1721, 318, 2117, 673, 2119, 2120, 668, 1721, 663, 670, - /* 2280 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2115, 2134, - /* 2290 */ 1721, 1721, 2151, 1721, 1721, 319, 2117, 673, 2119, 2120, - /* 2300 */ 668, 2084, 663, 669, 1721, 2134, 1721, 1721, 1721, 1721, - /* 2310 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2084, 1721, 669, - /* 2320 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2116, - /* 2330 */ 1721, 1721, 1721, 1721, 2115, 1721, 1721, 1721, 2151, 670, - /* 2340 */ 1721, 325, 2117, 673, 2119, 2120, 668, 1721, 663, 1721, - /* 2350 */ 2115, 1721, 1721, 2116, 2151, 1721, 1721, 329, 2117, 673, - /* 2360 */ 2119, 2120, 668, 670, 663, 2134, 1721, 1721, 1721, 1721, - /* 2370 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2084, 1721, 669, - /* 2380 */ 2116, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2134, - /* 2390 */ 670, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2116, - /* 2400 */ 1721, 2084, 1721, 669, 1721, 1721, 1721, 1721, 1721, 670, - /* 2410 */ 2115, 1721, 1721, 1721, 2151, 1721, 2134, 321, 2117, 673, - /* 2420 */ 2119, 2120, 668, 1721, 663, 1721, 1721, 1721, 2084, 1721, - /* 2430 */ 669, 1721, 1721, 1721, 2115, 2134, 1721, 1721, 2151, 1721, - /* 2440 */ 1721, 330, 2117, 673, 2119, 2120, 668, 2084, 663, 669, - /* 2450 */ 2116, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2460 */ 670, 2115, 1721, 1721, 1721, 2151, 2116, 1721, 322, 2117, - /* 2470 */ 673, 2119, 2120, 668, 1721, 663, 670, 1721, 1721, 1721, - /* 2480 */ 2115, 1721, 1721, 1721, 2151, 1721, 2134, 331, 2117, 673, - /* 2490 */ 2119, 2120, 668, 1721, 663, 1721, 2116, 1721, 2084, 1721, - /* 2500 */ 669, 1721, 2134, 1721, 1721, 1721, 670, 1721, 1721, 1721, - /* 2510 */ 1721, 1721, 1721, 1721, 2084, 1721, 669, 2116, 1721, 1721, - /* 2520 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 670, 1721, 1721, - /* 2530 */ 1721, 2115, 2134, 1721, 1721, 2151, 1721, 1721, 323, 2117, - /* 2540 */ 673, 2119, 2120, 668, 2084, 663, 669, 2115, 1721, 1721, - /* 2550 */ 1721, 2151, 1721, 2134, 336, 2117, 673, 2119, 2120, 668, - /* 2560 */ 1721, 663, 1721, 2116, 1721, 2084, 1721, 669, 1721, 1721, - /* 2570 */ 1721, 1721, 1721, 670, 1721, 1721, 1721, 2115, 1721, 1721, - /* 2580 */ 1721, 2151, 1721, 1721, 337, 2117, 673, 2119, 2120, 668, - /* 2590 */ 1721, 663, 1721, 1721, 1721, 1721, 1721, 1721, 2115, 2134, - /* 2600 */ 1721, 1721, 2151, 1721, 1721, 2128, 2117, 673, 2119, 2120, - /* 2610 */ 668, 2084, 663, 669, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2620 */ 1721, 1721, 1721, 2116, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2630 */ 1721, 1721, 1721, 670, 1721, 1721, 2116, 1721, 1721, 1721, - /* 2640 */ 1721, 1721, 1721, 1721, 2115, 1721, 670, 1721, 2151, 1721, - /* 2650 */ 1721, 2127, 2117, 673, 2119, 2120, 668, 1721, 663, 2134, - /* 2660 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2670 */ 1721, 2084, 2134, 669, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2680 */ 1721, 1721, 1721, 1721, 2084, 1721, 669, 2116, 1721, 1721, - /* 2690 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 670, 1721, 1721, - /* 2700 */ 1721, 1721, 1721, 1721, 2115, 1721, 1721, 1721, 2151, 1721, - /* 2710 */ 1721, 2126, 2117, 673, 2119, 2120, 668, 2115, 663, 1721, - /* 2720 */ 1721, 2151, 1721, 2134, 351, 2117, 673, 2119, 2120, 668, - /* 2730 */ 1721, 663, 1721, 2116, 1721, 2084, 1721, 669, 1721, 1721, - /* 2740 */ 1721, 1721, 1721, 670, 1721, 1721, 1721, 1721, 2116, 1721, - /* 2750 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 670, 1721, - /* 2760 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2115, 2134, - /* 2770 */ 1721, 1721, 2151, 1721, 1721, 352, 2117, 673, 2119, 2120, - /* 2780 */ 668, 2084, 663, 669, 2134, 1721, 1721, 1721, 1721, 1721, - /* 2790 */ 1721, 1721, 1721, 1721, 2116, 1721, 2084, 1721, 669, 1721, - /* 2800 */ 1721, 1721, 1721, 1721, 670, 1721, 1721, 1721, 1721, 2116, - /* 2810 */ 1721, 1721, 1721, 1721, 2115, 1721, 1721, 1721, 2151, 670, - /* 2820 */ 1721, 348, 2117, 673, 2119, 2120, 668, 1721, 663, 2115, - /* 2830 */ 2134, 1721, 1721, 2151, 1721, 1721, 353, 2117, 673, 2119, - /* 2840 */ 2120, 668, 2084, 663, 669, 2134, 1721, 1721, 1721, 1721, - /* 2850 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2084, 1721, 669, + /* 690 */ 202, 201, 69, 393, 639, 1495, 2003, 2116, 611, 1171, + /* 700 */ 1974, 1174, 1175, 557, 556, 555, 1576, 633, 1493, 365, + /* 710 */ 547, 137, 551, 477, 641, 698, 550, 618, 655, 1892, + /* 720 */ 2275, 549, 554, 368, 367, 630, 140, 548, 81, 80, + /* 730 */ 437, 1643, 1571, 193, 2134, 617, 186, 439, 1938, 1939, + /* 740 */ 2276, 619, 717, 244, 1944, 1501, 2084, 243, 671, 655, + /* 750 */ 1892, 383, 14, 13, 341, 655, 1892, 423, 1877, 1942, + /* 760 */ 421, 417, 413, 410, 430, 650, 1743, 2003, 448, 1944, + /* 770 */ 758, 106, 2077, 47, 463, 366, 388, 364, 363, 2115, + /* 780 */ 541, 103, 2151, 2116, 1942, 110, 2117, 675, 2119, 2120, + /* 790 */ 670, 1944, 665, 672, 1869, 655, 1892, 183, 397, 2204, + /* 800 */ 101, 543, 189, 389, 2200, 542, 1942, 90, 347, 1578, + /* 810 */ 1579, 372, 87, 573, 464, 2084, 188, 2078, 593, 1735, + /* 820 */ 2134, 2275, 553, 552, 2230, 1885, 277, 2212, 629, 362, + /* 830 */ 134, 628, 2084, 2275, 671, 1742, 2281, 186, 1887, 1551, + /* 840 */ 1561, 2276, 619, 630, 140, 1577, 1580, 245, 617, 186, + /* 850 */ 655, 1892, 1619, 2276, 619, 458, 1586, 2219, 12, 1496, + /* 860 */ 10, 1494, 1520, 593, 457, 2115, 2275, 1741, 2151, 517, + /* 870 */ 1879, 111, 2117, 675, 2119, 2120, 670, 657, 665, 2176, + /* 880 */ 622, 2281, 186, 2215, 2084, 2204, 2276, 619, 1499, 1500, + /* 890 */ 2201, 1550, 1553, 1554, 1555, 1556, 1557, 1558, 1559, 1560, + /* 900 */ 667, 663, 1569, 1570, 1572, 1573, 1574, 1575, 2, 46, + /* 910 */ 44, 659, 1523, 2176, 1468, 1469, 2084, 393, 427, 1495, + /* 920 */ 52, 2116, 3, 705, 155, 154, 702, 701, 700, 152, + /* 930 */ 1576, 633, 1493, 39, 38, 655, 1892, 45, 43, 42, + /* 940 */ 41, 40, 429, 425, 187, 2212, 2213, 2116, 138, 2217, + /* 950 */ 198, 1875, 655, 1892, 1889, 578, 1571, 672, 2134, 2238, + /* 960 */ 705, 155, 154, 702, 701, 700, 152, 399, 1520, 1501, + /* 970 */ 2084, 247, 671, 655, 1892, 165, 42, 41, 40, 2219, + /* 980 */ 1642, 39, 38, 1894, 2134, 45, 43, 42, 41, 40, + /* 990 */ 84, 543, 589, 83, 758, 542, 2084, 47, 671, 655, + /* 1000 */ 1892, 593, 1740, 2115, 2275, 2214, 2151, 2116, 1552, 110, + /* 1010 */ 2117, 675, 2119, 2120, 670, 2067, 665, 672, 634, 2281, + /* 1020 */ 186, 183, 9, 2204, 2276, 619, 1739, 389, 2200, 2115, + /* 1030 */ 1867, 87, 2151, 1578, 1579, 110, 2117, 675, 2119, 2120, + /* 1040 */ 670, 2094, 665, 254, 2134, 1738, 571, 2295, 2231, 2204, + /* 1050 */ 146, 2084, 135, 389, 2200, 1883, 2084, 1888, 671, 569, + /* 1060 */ 256, 567, 1989, 1551, 1561, 2098, 34, 655, 1892, 1577, + /* 1070 */ 1580, 1607, 39, 38, 1662, 2084, 45, 43, 42, 41, + /* 1080 */ 40, 729, 727, 1496, 1896, 1494, 638, 655, 1892, 2115, + /* 1090 */ 655, 1892, 2151, 164, 2084, 169, 2117, 675, 2119, 2120, + /* 1100 */ 670, 2100, 665, 1174, 1175, 312, 293, 200, 1921, 652, + /* 1110 */ 623, 665, 1499, 1500, 1552, 1550, 1553, 1554, 1555, 1556, + /* 1120 */ 1557, 1558, 1559, 1560, 667, 663, 1569, 1570, 1572, 1573, + /* 1130 */ 1574, 1575, 2, 46, 44, 1737, 2242, 2116, 655, 1892, + /* 1140 */ 1736, 393, 1733, 1495, 2094, 33, 1732, 672, 1731, 2251, + /* 1150 */ 655, 1892, 655, 1892, 1576, 1612, 1493, 653, 2102, 705, + /* 1160 */ 155, 154, 702, 701, 700, 152, 1730, 142, 2098, 299, + /* 1170 */ 2175, 400, 2224, 1639, 2134, 1944, 703, 1177, 704, 1935, + /* 1180 */ 1571, 1935, 1729, 1519, 2084, 2094, 2084, 1728, 671, 2084, + /* 1190 */ 1943, 2084, 2094, 1501, 1870, 2084, 625, 2084, 1727, 2102, + /* 1200 */ 1726, 1725, 2070, 251, 2100, 377, 2103, 718, 234, 2098, + /* 1210 */ 1854, 232, 74, 592, 665, 2084, 2098, 236, 758, 2115, + /* 1220 */ 235, 15, 2151, 2116, 422, 110, 2117, 675, 2119, 2120, + /* 1230 */ 670, 2084, 665, 672, 1639, 600, 2084, 2295, 148, 2204, + /* 1240 */ 153, 441, 546, 389, 2200, 2100, 390, 2084, 1781, 2084, + /* 1250 */ 2084, 415, 2100, 238, 442, 665, 237, 1578, 1579, 1774, + /* 1260 */ 2134, 82, 665, 240, 1260, 153, 239, 1772, 711, 575, + /* 1270 */ 558, 574, 2084, 662, 671, 64, 255, 64, 261, 1717, + /* 1280 */ 1718, 560, 14, 13, 666, 2105, 1834, 1551, 1561, 563, + /* 1290 */ 1240, 2244, 274, 1577, 1580, 227, 268, 1463, 606, 1832, + /* 1300 */ 1504, 1503, 712, 1831, 153, 2115, 48, 1496, 2151, 1494, + /* 1310 */ 2135, 110, 2117, 675, 2119, 2120, 670, 286, 665, 91, + /* 1320 */ 71, 151, 1466, 2295, 1238, 2204, 406, 64, 48, 389, + /* 1330 */ 2200, 1998, 1671, 53, 1670, 263, 1499, 1500, 2107, 1550, + /* 1340 */ 1553, 1554, 1555, 1556, 1557, 1558, 1559, 1560, 667, 663, + /* 1350 */ 1569, 1570, 1572, 1573, 1574, 1575, 2, 2116, 402, 401, + /* 1360 */ 1221, 637, 153, 1423, 48, 679, 1768, 672, 1509, 2269, + /* 1370 */ 151, 1764, 153, 1759, 291, 1932, 2234, 647, 295, 1576, + /* 1380 */ 136, 1502, 631, 2116, 1613, 1562, 276, 151, 273, 1, + /* 1390 */ 5, 409, 356, 672, 2134, 2223, 414, 1222, 1446, 306, + /* 1400 */ 199, 443, 1523, 1999, 480, 1571, 2084, 447, 671, 452, + /* 1410 */ 1518, 490, 465, 1991, 479, 472, 753, 491, 1501, 1305, + /* 1420 */ 2134, 311, 1332, 1597, 481, 488, 203, 1336, 626, 1343, + /* 1430 */ 2116, 204, 2084, 493, 671, 494, 206, 1341, 496, 2115, + /* 1440 */ 672, 498, 2151, 661, 156, 110, 2117, 675, 2119, 2120, + /* 1450 */ 670, 2116, 665, 1507, 1506, 1524, 4, 2295, 499, 2204, + /* 1460 */ 507, 672, 1526, 389, 2200, 2115, 500, 2134, 2151, 508, + /* 1470 */ 510, 110, 2117, 675, 2119, 2120, 670, 1521, 665, 2084, + /* 1480 */ 1525, 671, 214, 2295, 511, 2204, 512, 216, 2134, 389, + /* 1490 */ 2200, 1527, 513, 1194, 219, 515, 221, 85, 86, 519, + /* 1500 */ 2084, 536, 671, 225, 538, 537, 112, 346, 540, 1882, + /* 1510 */ 231, 1878, 2115, 2060, 577, 2151, 2057, 2056, 170, 2117, + /* 1520 */ 675, 2119, 2120, 670, 579, 665, 233, 89, 149, 307, + /* 1530 */ 158, 159, 1510, 2115, 1505, 248, 2151, 1880, 1876, 110, + /* 1540 */ 2117, 675, 2119, 2120, 670, 160, 665, 161, 583, 252, + /* 1550 */ 582, 2179, 1453, 2204, 584, 2116, 597, 389, 2200, 607, + /* 1560 */ 587, 1513, 1515, 250, 2250, 672, 590, 645, 259, 588, + /* 1570 */ 2297, 2235, 2245, 603, 663, 1569, 1570, 1572, 1573, 1574, + /* 1580 */ 1575, 262, 2249, 8, 616, 378, 610, 2226, 598, 596, + /* 1590 */ 595, 267, 2134, 379, 139, 272, 1639, 2298, 627, 1522, + /* 1600 */ 624, 2220, 382, 635, 2084, 269, 671, 1528, 2004, 308, + /* 1610 */ 643, 281, 96, 309, 644, 648, 98, 2018, 2017, 2016, + /* 1620 */ 649, 2116, 173, 385, 270, 1893, 271, 100, 61, 102, + /* 1630 */ 2185, 672, 1936, 313, 1855, 310, 754, 2115, 302, 755, + /* 1640 */ 2151, 757, 348, 110, 2117, 675, 2119, 2120, 670, 2116, + /* 1650 */ 665, 2274, 677, 275, 337, 2177, 51, 2204, 2134, 672, + /* 1660 */ 322, 389, 2200, 336, 326, 317, 2076, 2075, 349, 2074, + /* 1670 */ 2084, 315, 671, 78, 2071, 411, 412, 1486, 1487, 192, + /* 1680 */ 416, 2116, 2069, 418, 419, 420, 2134, 2068, 357, 2066, + /* 1690 */ 424, 672, 426, 2064, 428, 79, 1449, 1448, 2084, 2065, + /* 1700 */ 671, 2030, 2029, 2115, 2028, 435, 2151, 436, 2027, 110, + /* 1710 */ 2117, 675, 2119, 2120, 670, 2026, 665, 1982, 2134, 1400, + /* 1720 */ 1981, 658, 1979, 2204, 1978, 145, 1977, 389, 2200, 1980, + /* 1730 */ 2084, 2115, 671, 1976, 2151, 1975, 1973, 111, 2117, 675, + /* 1740 */ 2119, 2120, 670, 1972, 665, 1971, 197, 453, 1970, 455, + /* 1750 */ 1984, 2204, 2116, 1969, 1968, 2203, 2200, 483, 147, 1954, + /* 1760 */ 1953, 1952, 672, 2115, 1967, 1966, 2151, 1965, 1964, 111, + /* 1770 */ 2117, 675, 2119, 2120, 670, 1963, 665, 1962, 1961, 1960, + /* 1780 */ 2116, 1959, 1958, 2204, 1957, 1956, 1955, 660, 2200, 2134, + /* 1790 */ 672, 1983, 1951, 1950, 1402, 1949, 1948, 1947, 1946, 2116, + /* 1800 */ 1945, 2084, 1276, 671, 344, 1280, 345, 1797, 205, 669, + /* 1810 */ 1796, 1272, 1795, 1793, 207, 2116, 1754, 2134, 180, 1176, + /* 1820 */ 2104, 208, 1753, 2047, 2037, 672, 76, 2025, 77, 2084, + /* 1830 */ 210, 671, 2024, 218, 673, 220, 2134, 2151, 2002, 1871, + /* 1840 */ 111, 2117, 675, 2119, 2120, 670, 181, 665, 2084, 505, + /* 1850 */ 671, 212, 2134, 1792, 2204, 1790, 1214, 384, 351, 2200, + /* 1860 */ 520, 1788, 2115, 521, 2084, 2151, 671, 522, 328, 2117, + /* 1870 */ 675, 2119, 2120, 670, 526, 665, 2116, 524, 525, 1786, + /* 1880 */ 528, 2115, 529, 530, 2151, 1784, 672, 334, 2117, 675, + /* 1890 */ 2119, 2120, 670, 532, 665, 533, 2170, 2115, 581, 2116, + /* 1900 */ 2151, 534, 1771, 335, 2117, 675, 2119, 2120, 670, 672, + /* 1910 */ 665, 615, 1770, 2134, 1750, 1873, 761, 63, 392, 1348, + /* 1920 */ 1347, 1872, 230, 1263, 1261, 2084, 1259, 671, 1258, 1257, + /* 1930 */ 305, 1256, 1250, 1782, 1255, 369, 2134, 561, 1252, 726, + /* 1940 */ 1251, 394, 1249, 1775, 728, 370, 178, 1773, 2084, 371, + /* 1950 */ 671, 564, 751, 747, 743, 739, 303, 566, 2115, 1749, + /* 1960 */ 1748, 2151, 1747, 568, 335, 2117, 675, 2119, 2120, 670, + /* 1970 */ 570, 665, 113, 1473, 1475, 1472, 1477, 27, 2046, 1455, + /* 1980 */ 67, 2115, 2036, 1457, 2151, 56, 585, 335, 2117, 675, + /* 1990 */ 2119, 2120, 670, 2116, 665, 163, 108, 2023, 586, 296, + /* 2000 */ 2021, 253, 1459, 672, 375, 2280, 29, 20, 17, 591, + /* 2010 */ 1687, 258, 59, 6, 7, 599, 601, 60, 265, 260, + /* 2020 */ 1669, 31, 266, 2105, 171, 2116, 264, 21, 65, 30, + /* 2030 */ 2134, 651, 1661, 1707, 92, 672, 1708, 22, 1702, 278, + /* 2040 */ 1701, 380, 2084, 2116, 671, 1706, 1705, 381, 176, 2022, + /* 2050 */ 1636, 1635, 58, 672, 2020, 2019, 2001, 18, 94, 95, + /* 2060 */ 284, 646, 2134, 2000, 103, 97, 283, 23, 57, 297, + /* 2070 */ 285, 282, 1667, 294, 2084, 576, 671, 24, 2151, 287, + /* 2080 */ 2134, 330, 2117, 675, 2119, 2120, 670, 292, 665, 68, + /* 2090 */ 2116, 249, 2084, 11, 671, 99, 1588, 1587, 13, 1511, + /* 2100 */ 672, 1543, 1598, 678, 177, 190, 395, 2115, 1566, 2154, + /* 2110 */ 2151, 682, 1325, 319, 2117, 675, 2119, 2120, 670, 664, + /* 2120 */ 665, 685, 1564, 37, 16, 2115, 676, 2134, 2151, 1563, + /* 2130 */ 25, 320, 2117, 675, 2119, 2120, 670, 2116, 665, 2084, + /* 2140 */ 1535, 671, 674, 26, 688, 1333, 680, 672, 1330, 1327, + /* 2150 */ 683, 686, 691, 1321, 1319, 300, 1324, 1323, 1322, 2116, + /* 2160 */ 689, 692, 104, 1342, 105, 75, 1338, 1212, 706, 672, + /* 2170 */ 1244, 1243, 2115, 301, 2134, 2151, 1242, 1241, 321, 2117, + /* 2180 */ 675, 2119, 2120, 670, 2116, 665, 2084, 1239, 671, 1237, + /* 2190 */ 1236, 1235, 1270, 716, 672, 1233, 2134, 1232, 1231, 1230, + /* 2200 */ 1229, 1228, 1227, 1267, 1265, 1224, 2116, 1223, 2084, 1220, + /* 2210 */ 671, 1219, 1218, 1217, 1789, 736, 672, 737, 738, 2115, + /* 2220 */ 1787, 2134, 2151, 740, 741, 327, 2117, 675, 2119, 2120, + /* 2230 */ 670, 742, 665, 2084, 1785, 671, 744, 746, 745, 1783, + /* 2240 */ 748, 2115, 749, 2134, 2151, 750, 1769, 331, 2117, 675, + /* 2250 */ 2119, 2120, 670, 752, 665, 2084, 1166, 671, 304, 756, + /* 2260 */ 1721, 1721, 1497, 314, 759, 760, 2115, 1746, 1721, 2151, + /* 2270 */ 2116, 1721, 323, 2117, 675, 2119, 2120, 670, 1721, 665, + /* 2280 */ 672, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2115, 1721, + /* 2290 */ 2116, 2151, 1721, 1721, 332, 2117, 675, 2119, 2120, 670, + /* 2300 */ 672, 665, 1721, 1721, 1721, 2116, 1721, 2134, 1721, 1721, + /* 2310 */ 1721, 1721, 1721, 1721, 1721, 672, 1721, 1721, 1721, 2084, + /* 2320 */ 1721, 671, 1721, 1721, 1721, 1721, 1721, 2134, 1721, 1721, + /* 2330 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2116, 1721, 2084, + /* 2340 */ 1721, 671, 2134, 1721, 1721, 1721, 1721, 672, 1721, 1721, + /* 2350 */ 1721, 1721, 2115, 1721, 2084, 2151, 671, 1721, 324, 2117, + /* 2360 */ 675, 2119, 2120, 670, 1721, 665, 1721, 1721, 1721, 1721, + /* 2370 */ 1721, 1721, 2115, 1721, 2134, 2151, 1721, 1721, 333, 2117, + /* 2380 */ 675, 2119, 2120, 670, 1721, 665, 2084, 2115, 671, 1721, + /* 2390 */ 2151, 1721, 1721, 325, 2117, 675, 2119, 2120, 670, 1721, + /* 2400 */ 665, 1721, 1721, 1721, 1721, 2116, 1721, 1721, 1721, 1721, + /* 2410 */ 1721, 1721, 1721, 1721, 1721, 672, 1721, 1721, 1721, 2115, + /* 2420 */ 1721, 1721, 2151, 1721, 2116, 338, 2117, 675, 2119, 2120, + /* 2430 */ 670, 1721, 665, 1721, 672, 1721, 1721, 1721, 1721, 1721, + /* 2440 */ 1721, 1721, 2134, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2450 */ 1721, 1721, 2116, 1721, 2084, 1721, 671, 1721, 1721, 1721, + /* 2460 */ 1721, 2134, 672, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2470 */ 1721, 2116, 1721, 2084, 1721, 671, 1721, 1721, 1721, 1721, + /* 2480 */ 1721, 672, 1721, 1721, 1721, 1721, 1721, 2115, 1721, 2134, + /* 2490 */ 2151, 1721, 1721, 339, 2117, 675, 2119, 2120, 670, 1721, + /* 2500 */ 665, 2084, 1721, 671, 1721, 1721, 2115, 1721, 2134, 2151, + /* 2510 */ 1721, 1721, 2128, 2117, 675, 2119, 2120, 670, 2116, 665, + /* 2520 */ 2084, 1721, 671, 1721, 1721, 1721, 1721, 1721, 672, 1721, + /* 2530 */ 1721, 1721, 1721, 1721, 2115, 1721, 1721, 2151, 1721, 1721, + /* 2540 */ 2127, 2117, 675, 2119, 2120, 670, 1721, 665, 1721, 1721, + /* 2550 */ 1721, 1721, 1721, 2115, 1721, 2134, 2151, 1721, 1721, 2126, + /* 2560 */ 2117, 675, 2119, 2120, 670, 1721, 665, 2084, 1721, 671, + /* 2570 */ 1721, 1721, 1721, 1721, 1721, 2116, 1721, 1721, 1721, 1721, + /* 2580 */ 1721, 1721, 1721, 1721, 1721, 672, 1721, 1721, 1721, 1721, + /* 2590 */ 1721, 2116, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2600 */ 2115, 672, 1721, 2151, 1721, 1721, 353, 2117, 675, 2119, + /* 2610 */ 2120, 670, 2134, 665, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2620 */ 1721, 1721, 1721, 1721, 2084, 1721, 671, 1721, 2134, 1721, + /* 2630 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2640 */ 2084, 2116, 671, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2650 */ 1721, 672, 1721, 1721, 1721, 1721, 1721, 2115, 1721, 1721, + /* 2660 */ 2151, 2116, 1721, 354, 2117, 675, 2119, 2120, 670, 1721, + /* 2670 */ 665, 672, 1721, 2115, 1721, 1721, 2151, 1721, 2134, 350, + /* 2680 */ 2117, 675, 2119, 2120, 670, 1721, 665, 1721, 1721, 1721, + /* 2690 */ 2084, 1721, 671, 1721, 1721, 1721, 1721, 1721, 2134, 1721, + /* 2700 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2116, 1721, + /* 2710 */ 2084, 1721, 671, 1721, 1721, 1721, 1721, 1721, 672, 1721, + /* 2720 */ 1721, 1721, 1721, 2115, 1721, 1721, 2151, 1721, 1721, 355, + /* 2730 */ 2117, 675, 2119, 2120, 670, 1721, 665, 1721, 1721, 1721, + /* 2740 */ 1721, 1721, 229, 673, 1721, 2134, 2151, 1721, 1721, 330, + /* 2750 */ 2117, 675, 2119, 2120, 670, 1721, 665, 2084, 172, 671, + /* 2760 */ 1721, 1721, 1721, 1721, 535, 531, 527, 523, 226, 1721, + /* 2770 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2780 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2790 */ 2115, 1721, 1721, 2151, 1721, 1721, 329, 2117, 675, 2119, + /* 2800 */ 2120, 670, 1721, 665, 1721, 1721, 1721, 1721, 88, 1721, + /* 2810 */ 1721, 224, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2820 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2830 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2840 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2850 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, /* 2860 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2870 */ 1721, 1721, 1721, 1721, 1721, 671, 1721, 1721, 1721, 2151, - /* 2880 */ 1721, 1721, 328, 2117, 673, 2119, 2120, 668, 1721, 663, - /* 2890 */ 2115, 1721, 1721, 1721, 2151, 1721, 1721, 327, 2117, 673, - /* 2900 */ 2119, 2120, 668, 1721, 663, + /* 2870 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 223, 217, + /* 2880 */ 1721, 1721, 1721, 222, 1721, 514, 1721, 1721, 1721, 1721, + /* 2890 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2900 */ 1721, 1721, 1721, 215, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 333, 370, 383, 337, 400, 370, 340, 341, 404, 371, - /* 10 */ 343, 365, 12, 13, 14, 0, 371, 398, 399, 381, - /* 20 */ 20, 14, 22, 20, 8, 9, 381, 20, 12, 13, - /* 30 */ 14, 15, 16, 33, 0, 35, 369, 383, 20, 24, - /* 40 */ 25, 26, 27, 28, 29, 30, 31, 32, 381, 395, - /* 50 */ 383, 447, 398, 399, 450, 417, 418, 419, 412, 59, - /* 60 */ 44, 369, 417, 418, 419, 65, 428, 20, 376, 465, - /* 70 */ 466, 20, 72, 428, 470, 471, 384, 342, 343, 0, - /* 80 */ 337, 414, 348, 340, 341, 418, 436, 437, 421, 422, - /* 90 */ 423, 424, 425, 426, 332, 428, 334, 97, 364, 39, - /* 100 */ 100, 67, 68, 69, 70, 71, 372, 73, 74, 75, + /* 0 */ 333, 384, 371, 362, 401, 332, 401, 334, 405, 343, + /* 10 */ 343, 370, 12, 13, 14, 0, 399, 400, 377, 378, + /* 20 */ 20, 14, 22, 342, 8, 9, 385, 20, 12, 13, + /* 30 */ 14, 15, 16, 33, 0, 35, 21, 370, 20, 24, + /* 40 */ 25, 26, 27, 28, 29, 30, 31, 32, 3, 382, + /* 50 */ 447, 384, 447, 450, 388, 450, 8, 9, 351, 59, + /* 60 */ 12, 13, 14, 15, 16, 65, 359, 386, 465, 466, + /* 70 */ 465, 466, 72, 470, 471, 470, 471, 12, 13, 14, + /* 80 */ 15, 16, 415, 337, 20, 418, 340, 341, 421, 422, + /* 90 */ 423, 424, 425, 426, 369, 428, 20, 97, 436, 437, + /* 100 */ 100, 67, 68, 69, 70, 71, 381, 73, 74, 75, /* 110 */ 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, /* 120 */ 86, 87, 88, 89, 90, 91, 92, 93, 461, 462, - /* 130 */ 379, 420, 97, 382, 383, 21, 136, 137, 24, 25, + /* 130 */ 337, 20, 97, 340, 341, 21, 136, 137, 24, 25, /* 140 */ 26, 27, 28, 29, 30, 31, 32, 112, 113, 114, - /* 150 */ 115, 116, 117, 118, 119, 120, 121, 446, 123, 124, - /* 160 */ 125, 126, 127, 128, 369, 20, 166, 167, 447, 166, - /* 170 */ 167, 450, 172, 173, 369, 342, 343, 442, 443, 444, - /* 180 */ 445, 376, 447, 448, 389, 390, 186, 466, 188, 384, - /* 190 */ 330, 470, 471, 360, 130, 131, 64, 8, 9, 135, - /* 200 */ 367, 12, 13, 14, 15, 16, 59, 100, 129, 130, - /* 210 */ 131, 132, 133, 134, 135, 215, 216, 0, 218, 219, + /* 150 */ 115, 116, 117, 118, 119, 120, 121, 370, 123, 124, + /* 160 */ 125, 126, 127, 128, 100, 20, 166, 167, 20, 420, + /* 170 */ 342, 343, 172, 173, 330, 8, 9, 390, 391, 12, + /* 180 */ 13, 14, 15, 16, 166, 167, 186, 20, 188, 361, + /* 190 */ 342, 343, 362, 100, 178, 446, 368, 8, 9, 59, + /* 200 */ 370, 12, 13, 14, 15, 16, 8, 9, 378, 361, + /* 210 */ 12, 13, 14, 15, 16, 215, 216, 0, 218, 219, /* 220 */ 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, - /* 230 */ 230, 231, 232, 233, 234, 235, 12, 13, 181, 4, - /* 240 */ 20, 18, 100, 20, 20, 100, 22, 100, 20, 102, - /* 250 */ 27, 333, 447, 30, 65, 450, 33, 33, 369, 35, - /* 260 */ 400, 343, 205, 206, 404, 12, 13, 14, 15, 16, - /* 270 */ 465, 466, 49, 100, 51, 470, 471, 54, 43, 390, - /* 280 */ 45, 46, 368, 59, 67, 68, 69, 369, 0, 65, - /* 290 */ 100, 74, 75, 76, 380, 20, 72, 80, 109, 381, - /* 300 */ 110, 383, 85, 86, 87, 88, 20, 447, 91, 21, - /* 310 */ 450, 251, 24, 25, 26, 27, 28, 29, 30, 31, - /* 320 */ 32, 97, 99, 348, 100, 465, 466, 185, 3, 187, - /* 330 */ 470, 471, 414, 20, 111, 22, 418, 342, 343, 421, - /* 340 */ 422, 423, 424, 425, 426, 20, 428, 372, 35, 431, - /* 350 */ 342, 433, 434, 435, 165, 360, 214, 439, 440, 252, - /* 360 */ 136, 137, 367, 140, 20, 52, 143, 144, 145, 146, + /* 230 */ 230, 231, 232, 233, 234, 235, 12, 13, 0, 370, + /* 240 */ 100, 18, 102, 20, 20, 401, 22, 100, 100, 405, + /* 250 */ 27, 333, 447, 30, 65, 450, 33, 33, 343, 35, + /* 260 */ 391, 343, 24, 25, 26, 27, 28, 29, 30, 31, + /* 270 */ 32, 466, 49, 20, 51, 470, 471, 54, 185, 362, + /* 280 */ 187, 64, 447, 59, 22, 450, 338, 370, 370, 65, + /* 290 */ 342, 447, 344, 277, 450, 378, 72, 35, 109, 101, + /* 300 */ 382, 466, 384, 388, 384, 470, 471, 214, 447, 465, + /* 310 */ 466, 450, 64, 333, 470, 471, 396, 272, 4, 399, + /* 320 */ 400, 97, 99, 343, 100, 345, 465, 466, 35, 342, + /* 330 */ 343, 470, 471, 415, 111, 168, 418, 342, 343, 421, + /* 340 */ 422, 423, 424, 425, 426, 252, 428, 20, 361, 431, + /* 350 */ 370, 433, 434, 435, 165, 368, 361, 439, 440, 97, + /* 360 */ 136, 137, 382, 140, 384, 72, 143, 144, 145, 146, /* 370 */ 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, - /* 380 */ 157, 158, 159, 338, 161, 162, 163, 342, 168, 344, - /* 390 */ 166, 167, 447, 385, 252, 450, 172, 173, 14, 252, - /* 400 */ 67, 68, 69, 64, 20, 382, 383, 74, 75, 76, - /* 410 */ 186, 466, 188, 80, 171, 470, 471, 0, 85, 86, - /* 420 */ 87, 88, 136, 137, 91, 252, 101, 35, 239, 240, + /* 380 */ 157, 158, 159, 20, 161, 162, 163, 14, 20, 20, + /* 390 */ 166, 167, 252, 20, 13, 415, 172, 173, 418, 252, + /* 400 */ 252, 421, 422, 423, 424, 425, 426, 64, 428, 20, + /* 410 */ 186, 22, 188, 433, 100, 435, 35, 342, 343, 439, + /* 420 */ 440, 168, 100, 171, 342, 342, 343, 333, 239, 240, /* 430 */ 241, 242, 243, 244, 245, 246, 247, 248, 249, 215, - /* 440 */ 216, 166, 218, 219, 220, 221, 222, 223, 224, 225, + /* 440 */ 216, 52, 218, 219, 220, 221, 222, 223, 224, 225, /* 450 */ 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, - /* 460 */ 236, 12, 13, 33, 72, 22, 351, 20, 338, 20, - /* 470 */ 333, 22, 342, 358, 344, 215, 111, 20, 35, 49, - /* 480 */ 343, 64, 33, 81, 35, 55, 56, 57, 58, 59, - /* 490 */ 420, 8, 9, 168, 333, 12, 13, 14, 15, 16, - /* 500 */ 257, 258, 259, 20, 20, 4, 369, 371, 59, 355, - /* 510 */ 356, 72, 20, 343, 65, 72, 446, 381, 381, 342, - /* 520 */ 383, 72, 262, 263, 264, 265, 266, 267, 268, 99, - /* 530 */ 8, 9, 102, 236, 12, 13, 14, 15, 16, 369, - /* 540 */ 97, 369, 381, 141, 142, 13, 97, 333, 376, 100, - /* 550 */ 166, 414, 165, 417, 418, 418, 384, 100, 421, 422, - /* 560 */ 423, 424, 425, 426, 428, 428, 164, 35, 18, 392, - /* 570 */ 433, 394, 435, 23, 8, 9, 439, 440, 12, 13, - /* 580 */ 14, 15, 16, 136, 137, 136, 137, 37, 38, 452, - /* 590 */ 333, 41, 100, 423, 4, 381, 20, 460, 22, 169, - /* 600 */ 170, 100, 350, 342, 174, 236, 176, 238, 420, 19, - /* 610 */ 60, 61, 62, 63, 111, 166, 167, 361, 366, 172, - /* 620 */ 173, 172, 173, 33, 194, 369, 239, 375, 52, 186, - /* 630 */ 72, 188, 376, 377, 446, 186, 249, 188, 381, 49, - /* 640 */ 384, 0, 342, 343, 54, 342, 343, 369, 333, 59, - /* 650 */ 100, 168, 168, 392, 376, 394, 371, 129, 215, 216, - /* 660 */ 360, 133, 384, 360, 215, 216, 381, 218, 219, 220, + /* 460 */ 236, 12, 13, 136, 137, 22, 12, 13, 100, 20, + /* 470 */ 22, 22, 67, 68, 69, 393, 382, 395, 35, 74, + /* 480 */ 75, 76, 33, 35, 35, 80, 3, 333, 2, 35, + /* 490 */ 85, 86, 87, 88, 8, 9, 91, 343, 12, 13, + /* 500 */ 14, 15, 16, 20, 20, 136, 137, 333, 59, 257, + /* 510 */ 258, 259, 215, 379, 65, 72, 382, 343, 443, 444, + /* 520 */ 445, 72, 447, 448, 370, 442, 443, 444, 445, 380, + /* 530 */ 447, 448, 383, 384, 366, 20, 382, 22, 384, 166, + /* 540 */ 97, 172, 173, 20, 370, 97, 97, 8, 9, 100, + /* 550 */ 35, 12, 13, 14, 15, 16, 382, 44, 384, 262, + /* 560 */ 263, 264, 265, 266, 267, 268, 252, 52, 18, 415, + /* 570 */ 333, 111, 418, 23, 252, 421, 422, 423, 424, 425, + /* 580 */ 426, 413, 428, 44, 101, 136, 137, 37, 38, 415, + /* 590 */ 236, 41, 418, 358, 4, 421, 422, 423, 424, 425, + /* 600 */ 426, 427, 428, 429, 430, 342, 343, 372, 348, 19, + /* 610 */ 60, 61, 62, 63, 101, 166, 167, 382, 343, 382, + /* 620 */ 252, 172, 173, 33, 361, 365, 472, 473, 338, 186, + /* 630 */ 81, 188, 342, 373, 344, 186, 0, 188, 370, 49, + /* 640 */ 370, 370, 188, 342, 54, 370, 378, 377, 377, 59, + /* 650 */ 100, 168, 168, 418, 419, 385, 385, 72, 215, 216, + /* 660 */ 4, 130, 131, 428, 215, 216, 135, 218, 219, 220, /* 670 */ 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, /* 680 */ 231, 232, 233, 234, 235, 12, 13, 14, 138, 99, - /* 690 */ 369, 333, 102, 20, 370, 22, 381, 376, 342, 343, - /* 700 */ 178, 343, 417, 418, 419, 384, 33, 333, 35, 252, - /* 710 */ 357, 2, 359, 428, 342, 343, 360, 8, 9, 342, - /* 720 */ 343, 12, 13, 14, 15, 16, 358, 369, 178, 179, - /* 730 */ 180, 350, 59, 183, 168, 355, 356, 360, 342, 381, - /* 740 */ 370, 383, 8, 9, 252, 72, 12, 13, 14, 15, - /* 750 */ 16, 342, 343, 252, 204, 381, 375, 207, 0, 343, - /* 760 */ 210, 211, 212, 213, 214, 14, 333, 342, 343, 360, - /* 770 */ 97, 20, 414, 100, 342, 343, 418, 333, 333, 421, - /* 780 */ 422, 423, 424, 425, 426, 360, 428, 343, 392, 345, - /* 790 */ 394, 433, 360, 435, 2, 4, 333, 439, 440, 277, - /* 800 */ 8, 9, 252, 387, 12, 13, 14, 15, 16, 136, - /* 810 */ 137, 342, 343, 369, 381, 443, 444, 445, 460, 447, - /* 820 */ 448, 21, 450, 342, 343, 381, 381, 383, 370, 360, - /* 830 */ 20, 342, 343, 59, 34, 101, 36, 465, 466, 166, - /* 840 */ 167, 343, 470, 471, 381, 172, 173, 8, 9, 360, - /* 850 */ 209, 12, 13, 14, 15, 16, 0, 431, 414, 186, - /* 860 */ 434, 188, 418, 342, 343, 421, 422, 423, 424, 425, - /* 870 */ 426, 369, 428, 99, 131, 370, 102, 433, 135, 435, - /* 880 */ 37, 360, 369, 439, 440, 387, 384, 400, 215, 216, - /* 890 */ 377, 218, 219, 220, 221, 222, 223, 224, 225, 226, + /* 690 */ 141, 142, 102, 20, 393, 22, 395, 333, 423, 43, + /* 700 */ 0, 45, 46, 67, 68, 69, 33, 343, 35, 37, + /* 710 */ 74, 75, 76, 164, 342, 111, 80, 447, 342, 343, + /* 720 */ 450, 85, 86, 87, 88, 342, 343, 91, 178, 179, + /* 730 */ 180, 4, 59, 183, 370, 465, 466, 361, 383, 384, + /* 740 */ 470, 471, 72, 131, 370, 72, 382, 135, 384, 342, + /* 750 */ 343, 377, 1, 2, 204, 342, 343, 207, 371, 385, + /* 760 */ 210, 211, 212, 213, 214, 393, 333, 395, 361, 370, + /* 770 */ 97, 100, 401, 100, 361, 103, 377, 105, 106, 415, + /* 780 */ 108, 110, 418, 333, 385, 421, 422, 423, 424, 425, + /* 790 */ 426, 370, 428, 343, 0, 342, 343, 433, 377, 435, + /* 800 */ 348, 129, 252, 439, 440, 133, 385, 195, 196, 136, + /* 810 */ 137, 199, 350, 201, 361, 382, 452, 401, 447, 334, + /* 820 */ 370, 450, 355, 356, 460, 373, 443, 444, 445, 367, + /* 830 */ 447, 448, 382, 450, 384, 333, 465, 466, 376, 166, + /* 840 */ 167, 470, 471, 342, 343, 172, 173, 130, 465, 466, + /* 850 */ 342, 343, 101, 470, 471, 155, 14, 420, 236, 186, + /* 860 */ 238, 188, 20, 447, 164, 415, 450, 333, 418, 361, + /* 870 */ 371, 421, 422, 423, 424, 425, 426, 432, 428, 434, + /* 880 */ 44, 465, 466, 446, 382, 435, 470, 471, 215, 216, + /* 890 */ 440, 218, 219, 220, 221, 222, 223, 224, 225, 226, /* 900 */ 227, 228, 229, 230, 231, 232, 233, 234, 235, 12, - /* 910 */ 13, 333, 432, 155, 434, 342, 343, 20, 333, 22, - /* 920 */ 333, 343, 164, 345, 443, 444, 445, 59, 447, 448, - /* 930 */ 33, 333, 35, 360, 447, 45, 46, 450, 195, 196, - /* 940 */ 101, 343, 199, 432, 201, 434, 103, 369, 105, 106, - /* 950 */ 333, 108, 465, 466, 14, 334, 59, 470, 471, 381, - /* 960 */ 20, 383, 342, 343, 342, 343, 381, 369, 381, 72, - /* 970 */ 102, 22, 129, 14, 15, 16, 133, 3, 168, 381, - /* 980 */ 360, 383, 360, 371, 35, 129, 130, 131, 132, 133, - /* 990 */ 134, 135, 414, 381, 97, 400, 418, 100, 381, 421, - /* 1000 */ 422, 423, 424, 425, 426, 333, 428, 400, 1, 2, - /* 1010 */ 333, 433, 414, 435, 333, 343, 418, 439, 440, 421, - /* 1020 */ 422, 423, 424, 425, 426, 427, 428, 429, 430, 417, - /* 1030 */ 418, 8, 9, 136, 137, 12, 13, 14, 15, 16, - /* 1040 */ 428, 369, 447, 361, 253, 450, 97, 343, 44, 342, - /* 1050 */ 343, 369, 65, 381, 447, 383, 370, 450, 381, 377, - /* 1060 */ 465, 466, 381, 166, 167, 470, 471, 360, 378, 172, - /* 1070 */ 173, 381, 465, 466, 250, 251, 361, 470, 471, 136, - /* 1080 */ 137, 342, 343, 186, 369, 188, 414, 342, 343, 22, - /* 1090 */ 418, 387, 377, 421, 422, 423, 424, 425, 426, 360, - /* 1100 */ 428, 400, 35, 342, 343, 360, 166, 42, 101, 44, - /* 1110 */ 130, 378, 215, 216, 381, 218, 219, 220, 221, 222, + /* 910 */ 13, 432, 20, 434, 197, 198, 382, 20, 181, 22, + /* 920 */ 42, 333, 44, 129, 130, 131, 132, 133, 134, 135, + /* 930 */ 33, 343, 35, 8, 9, 342, 343, 12, 13, 14, + /* 940 */ 15, 16, 205, 206, 443, 444, 445, 333, 447, 448, + /* 950 */ 59, 371, 342, 343, 361, 401, 59, 343, 370, 345, + /* 960 */ 129, 130, 131, 132, 133, 134, 135, 362, 20, 72, + /* 970 */ 382, 361, 384, 342, 343, 370, 14, 15, 16, 420, + /* 980 */ 253, 8, 9, 378, 370, 12, 13, 14, 15, 16, + /* 990 */ 99, 129, 361, 102, 97, 133, 382, 100, 384, 342, + /* 1000 */ 343, 447, 333, 415, 450, 446, 418, 333, 166, 421, + /* 1010 */ 422, 423, 424, 425, 426, 0, 428, 343, 361, 465, + /* 1020 */ 466, 433, 39, 435, 470, 471, 333, 439, 440, 415, + /* 1030 */ 0, 350, 418, 136, 137, 421, 422, 423, 424, 425, + /* 1040 */ 426, 358, 428, 406, 370, 333, 21, 433, 460, 435, + /* 1050 */ 42, 382, 44, 439, 440, 372, 382, 376, 384, 34, + /* 1060 */ 168, 36, 343, 166, 167, 382, 2, 342, 343, 172, + /* 1070 */ 173, 165, 8, 9, 101, 382, 12, 13, 14, 15, + /* 1080 */ 16, 355, 356, 186, 371, 188, 361, 342, 343, 415, + /* 1090 */ 342, 343, 418, 168, 382, 421, 422, 423, 424, 425, + /* 1100 */ 426, 418, 428, 45, 46, 363, 361, 388, 366, 361, + /* 1110 */ 274, 428, 215, 216, 166, 218, 219, 220, 221, 222, /* 1120 */ 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, - /* 1130 */ 233, 234, 235, 12, 13, 342, 343, 342, 343, 333, - /* 1140 */ 333, 20, 361, 22, 472, 473, 333, 333, 447, 0, - /* 1150 */ 369, 450, 333, 360, 33, 360, 35, 343, 377, 345, - /* 1160 */ 44, 44, 22, 333, 97, 333, 465, 466, 0, 405, - /* 1170 */ 362, 470, 471, 365, 333, 35, 378, 197, 198, 381, - /* 1180 */ 59, 111, 104, 369, 343, 107, 345, 381, 381, 42, - /* 1190 */ 200, 44, 202, 72, 381, 381, 447, 383, 49, 450, - /* 1200 */ 381, 48, 215, 13, 443, 444, 445, 0, 447, 448, - /* 1210 */ 369, 381, 44, 381, 465, 466, 391, 101, 97, 470, - /* 1220 */ 471, 100, 381, 104, 383, 35, 107, 474, 414, 22, - /* 1230 */ 160, 35, 418, 44, 333, 421, 422, 423, 424, 425, - /* 1240 */ 426, 35, 428, 463, 343, 104, 272, 433, 107, 435, - /* 1250 */ 0, 0, 346, 439, 440, 414, 44, 136, 137, 418, - /* 1260 */ 44, 13, 421, 422, 423, 424, 425, 426, 104, 428, - /* 1270 */ 369, 107, 22, 22, 433, 44, 435, 44, 274, 457, - /* 1280 */ 439, 440, 381, 35, 383, 47, 369, 166, 167, 391, - /* 1290 */ 101, 35, 346, 172, 173, 341, 129, 130, 131, 132, - /* 1300 */ 133, 134, 135, 1, 2, 44, 343, 186, 44, 188, - /* 1310 */ 380, 44, 391, 101, 44, 414, 44, 101, 44, 418, - /* 1320 */ 449, 168, 421, 422, 423, 424, 425, 426, 72, 428, - /* 1330 */ 44, 0, 101, 44, 101, 44, 215, 216, 100, 218, + /* 1130 */ 233, 234, 235, 12, 13, 333, 462, 333, 342, 343, + /* 1140 */ 333, 20, 333, 22, 358, 239, 333, 343, 333, 345, + /* 1150 */ 342, 343, 342, 343, 33, 249, 35, 361, 372, 129, + /* 1160 */ 130, 131, 132, 133, 134, 135, 333, 431, 382, 361, + /* 1170 */ 434, 361, 250, 251, 370, 370, 379, 14, 379, 382, + /* 1180 */ 59, 382, 333, 20, 382, 358, 382, 333, 384, 382, + /* 1190 */ 385, 382, 358, 72, 0, 382, 44, 382, 333, 372, + /* 1200 */ 333, 333, 0, 371, 418, 419, 372, 357, 104, 382, + /* 1210 */ 360, 107, 111, 48, 428, 382, 382, 104, 97, 415, + /* 1220 */ 107, 100, 418, 333, 209, 421, 422, 423, 424, 425, + /* 1230 */ 426, 382, 428, 343, 251, 345, 382, 433, 44, 435, + /* 1240 */ 44, 22, 13, 439, 440, 418, 419, 382, 0, 382, + /* 1250 */ 382, 49, 418, 104, 35, 428, 107, 136, 137, 0, + /* 1260 */ 370, 160, 428, 104, 35, 44, 107, 0, 13, 200, + /* 1270 */ 22, 202, 382, 65, 384, 44, 59, 44, 44, 136, + /* 1280 */ 137, 22, 1, 2, 371, 47, 359, 166, 167, 22, + /* 1290 */ 35, 392, 474, 172, 173, 346, 457, 101, 463, 358, + /* 1300 */ 35, 35, 13, 358, 44, 415, 44, 186, 418, 188, + /* 1310 */ 370, 421, 422, 423, 424, 425, 426, 44, 428, 102, + /* 1320 */ 44, 44, 101, 433, 35, 435, 346, 44, 44, 439, + /* 1330 */ 440, 392, 101, 168, 101, 101, 215, 216, 100, 218, /* 1340 */ 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, - /* 1350 */ 229, 230, 231, 232, 233, 234, 235, 441, 467, 12, - /* 1360 */ 13, 44, 101, 462, 44, 101, 44, 333, 101, 22, - /* 1370 */ 44, 101, 44, 101, 13, 101, 451, 343, 254, 345, - /* 1380 */ 33, 50, 35, 416, 188, 49, 415, 101, 184, 388, - /* 1390 */ 101, 402, 101, 276, 188, 42, 35, 333, 20, 391, - /* 1400 */ 388, 165, 386, 369, 20, 342, 59, 343, 342, 345, - /* 1410 */ 388, 386, 386, 98, 354, 381, 342, 383, 101, 72, - /* 1420 */ 96, 101, 353, 101, 95, 352, 342, 101, 342, 101, - /* 1430 */ 342, 20, 335, 369, 48, 339, 335, 339, 20, 409, - /* 1440 */ 350, 20, 383, 350, 97, 381, 20, 383, 414, 344, - /* 1450 */ 20, 401, 418, 350, 344, 421, 422, 423, 424, 425, - /* 1460 */ 426, 350, 428, 350, 350, 333, 53, 433, 342, 435, - /* 1470 */ 350, 335, 347, 439, 440, 343, 347, 369, 414, 335, - /* 1480 */ 381, 369, 418, 381, 369, 421, 422, 423, 424, 425, - /* 1490 */ 426, 369, 428, 342, 413, 203, 369, 433, 100, 435, - /* 1500 */ 369, 369, 369, 439, 440, 411, 369, 369, 369, 381, - /* 1510 */ 348, 369, 408, 381, 191, 383, 409, 192, 383, 348, - /* 1520 */ 190, 342, 269, 261, 456, 407, 260, 381, 456, 406, - /* 1530 */ 396, 381, 177, 186, 391, 188, 391, 381, 381, 459, - /* 1540 */ 458, 456, 271, 270, 396, 454, 414, 455, 255, 273, - /* 1550 */ 418, 475, 275, 421, 422, 423, 424, 425, 426, 453, - /* 1560 */ 428, 278, 215, 216, 469, 433, 343, 435, 416, 251, - /* 1570 */ 20, 439, 440, 333, 420, 228, 229, 230, 231, 232, - /* 1580 */ 233, 234, 348, 343, 342, 344, 20, 348, 394, 396, - /* 1590 */ 381, 381, 381, 333, 381, 381, 396, 170, 468, 381, - /* 1600 */ 348, 393, 365, 343, 100, 348, 343, 438, 100, 369, - /* 1610 */ 381, 342, 36, 359, 0, 336, 373, 335, 403, 397, - /* 1620 */ 0, 381, 397, 383, 0, 331, 348, 410, 363, 369, - /* 1630 */ 349, 363, 42, 0, 35, 35, 208, 35, 363, 35, - /* 1640 */ 208, 381, 35, 383, 0, 35, 208, 0, 208, 0, - /* 1650 */ 35, 0, 22, 0, 414, 195, 35, 188, 418, 186, - /* 1660 */ 0, 421, 422, 423, 424, 425, 426, 0, 428, 0, - /* 1670 */ 182, 181, 0, 433, 414, 435, 47, 333, 418, 439, - /* 1680 */ 440, 421, 422, 423, 424, 425, 426, 343, 428, 0, - /* 1690 */ 0, 0, 0, 433, 42, 435, 0, 0, 0, 439, - /* 1700 */ 440, 333, 0, 0, 0, 0, 0, 155, 35, 0, - /* 1710 */ 155, 343, 0, 369, 0, 0, 0, 0, 0, 0, - /* 1720 */ 0, 0, 0, 0, 0, 381, 0, 383, 0, 0, - /* 1730 */ 0, 0, 0, 0, 0, 42, 0, 369, 0, 0, - /* 1740 */ 0, 0, 139, 0, 0, 0, 59, 22, 48, 381, - /* 1750 */ 0, 383, 0, 59, 59, 48, 0, 22, 414, 22, - /* 1760 */ 0, 14, 418, 177, 0, 421, 422, 423, 424, 425, - /* 1770 */ 426, 44, 428, 0, 0, 0, 0, 35, 333, 435, - /* 1780 */ 0, 0, 414, 439, 440, 39, 418, 39, 343, 421, - /* 1790 */ 422, 423, 424, 425, 426, 42, 428, 39, 47, 40, - /* 1800 */ 0, 333, 47, 435, 47, 0, 35, 439, 440, 39, - /* 1810 */ 49, 343, 0, 35, 369, 66, 49, 0, 39, 35, - /* 1820 */ 0, 49, 39, 35, 333, 39, 381, 0, 383, 0, - /* 1830 */ 0, 49, 0, 35, 343, 22, 0, 369, 35, 35, - /* 1840 */ 44, 35, 35, 35, 35, 35, 44, 333, 22, 381, - /* 1850 */ 35, 383, 0, 35, 35, 22, 0, 343, 0, 414, - /* 1860 */ 369, 107, 22, 418, 22, 51, 421, 422, 423, 424, - /* 1870 */ 425, 426, 381, 428, 383, 109, 35, 0, 35, 0, - /* 1880 */ 435, 35, 414, 369, 439, 440, 418, 0, 22, 421, - /* 1890 */ 422, 423, 424, 425, 426, 381, 428, 383, 20, 35, - /* 1900 */ 35, 100, 35, 435, 0, 414, 193, 35, 440, 418, - /* 1910 */ 333, 100, 421, 422, 423, 424, 425, 426, 22, 428, - /* 1920 */ 343, 0, 101, 22, 0, 0, 3, 44, 414, 101, - /* 1930 */ 168, 256, 418, 48, 100, 421, 422, 423, 424, 425, - /* 1940 */ 426, 48, 428, 175, 100, 168, 369, 98, 101, 96, - /* 1950 */ 44, 374, 44, 168, 189, 464, 101, 100, 381, 170, - /* 1960 */ 383, 100, 100, 44, 101, 47, 256, 3, 100, 47, - /* 1970 */ 44, 101, 101, 44, 35, 333, 35, 35, 35, 35, - /* 1980 */ 35, 101, 101, 0, 47, 343, 0, 473, 0, 0, - /* 1990 */ 44, 414, 100, 39, 47, 418, 333, 47, 421, 422, - /* 2000 */ 423, 424, 425, 426, 101, 428, 343, 100, 171, 333, - /* 2010 */ 101, 369, 100, 100, 100, 0, 39, 169, 47, 343, - /* 2020 */ 110, 2, 44, 381, 22, 383, 98, 100, 98, 237, - /* 2030 */ 22, 215, 369, 100, 47, 101, 101, 374, 100, 47, - /* 2040 */ 250, 101, 217, 111, 381, 369, 383, 256, 100, 100, - /* 2050 */ 374, 100, 35, 101, 101, 100, 414, 381, 35, 383, - /* 2060 */ 418, 1, 100, 421, 422, 423, 424, 425, 426, 101, - /* 2070 */ 428, 35, 430, 100, 35, 101, 100, 414, 35, 19, - /* 2080 */ 101, 418, 100, 35, 421, 422, 423, 424, 425, 426, - /* 2090 */ 414, 428, 101, 33, 418, 100, 333, 421, 422, 423, - /* 2100 */ 424, 425, 426, 122, 428, 100, 343, 122, 44, 49, - /* 2110 */ 100, 35, 122, 100, 22, 55, 56, 57, 58, 59, - /* 2120 */ 333, 122, 66, 65, 35, 35, 35, 72, 35, 35, - /* 2130 */ 343, 35, 369, 35, 35, 35, 35, 35, 94, 44, - /* 2140 */ 35, 35, 22, 35, 381, 35, 383, 35, 35, 72, - /* 2150 */ 35, 35, 22, 35, 35, 0, 369, 35, 35, 99, - /* 2160 */ 49, 0, 102, 39, 35, 39, 0, 35, 381, 0, - /* 2170 */ 383, 35, 0, 39, 49, 35, 49, 414, 39, 35, - /* 2180 */ 49, 418, 0, 333, 421, 422, 423, 424, 425, 426, - /* 2190 */ 22, 428, 21, 343, 134, 22, 22, 21, 20, 476, - /* 2200 */ 476, 414, 476, 476, 476, 418, 476, 333, 421, 422, - /* 2210 */ 423, 424, 425, 426, 476, 428, 476, 343, 476, 369, - /* 2220 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 169, - /* 2230 */ 476, 381, 476, 383, 174, 476, 476, 476, 476, 476, - /* 2240 */ 476, 476, 476, 369, 476, 476, 476, 476, 476, 476, - /* 2250 */ 476, 476, 476, 333, 194, 381, 476, 383, 476, 476, - /* 2260 */ 476, 476, 476, 343, 414, 476, 476, 476, 418, 333, - /* 2270 */ 476, 421, 422, 423, 424, 425, 426, 476, 428, 343, - /* 2280 */ 476, 476, 476, 476, 476, 476, 476, 476, 414, 369, - /* 2290 */ 476, 476, 418, 476, 476, 421, 422, 423, 424, 425, - /* 2300 */ 426, 381, 428, 383, 476, 369, 476, 476, 476, 476, - /* 2310 */ 476, 476, 476, 476, 476, 476, 476, 381, 476, 383, - /* 2320 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 333, - /* 2330 */ 476, 476, 476, 476, 414, 476, 476, 476, 418, 343, - /* 2340 */ 476, 421, 422, 423, 424, 425, 426, 476, 428, 476, - /* 2350 */ 414, 476, 476, 333, 418, 476, 476, 421, 422, 423, - /* 2360 */ 424, 425, 426, 343, 428, 369, 476, 476, 476, 476, - /* 2370 */ 476, 476, 476, 476, 476, 476, 476, 381, 476, 383, - /* 2380 */ 333, 476, 476, 476, 476, 476, 476, 476, 476, 369, - /* 2390 */ 343, 476, 476, 476, 476, 476, 476, 476, 476, 333, - /* 2400 */ 476, 381, 476, 383, 476, 476, 476, 476, 476, 343, - /* 2410 */ 414, 476, 476, 476, 418, 476, 369, 421, 422, 423, - /* 2420 */ 424, 425, 426, 476, 428, 476, 476, 476, 381, 476, - /* 2430 */ 383, 476, 476, 476, 414, 369, 476, 476, 418, 476, - /* 2440 */ 476, 421, 422, 423, 424, 425, 426, 381, 428, 383, - /* 2450 */ 333, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2460 */ 343, 414, 476, 476, 476, 418, 333, 476, 421, 422, - /* 2470 */ 423, 424, 425, 426, 476, 428, 343, 476, 476, 476, - /* 2480 */ 414, 476, 476, 476, 418, 476, 369, 421, 422, 423, - /* 2490 */ 424, 425, 426, 476, 428, 476, 333, 476, 381, 476, - /* 2500 */ 383, 476, 369, 476, 476, 476, 343, 476, 476, 476, - /* 2510 */ 476, 476, 476, 476, 381, 476, 383, 333, 476, 476, - /* 2520 */ 476, 476, 476, 476, 476, 476, 476, 343, 476, 476, - /* 2530 */ 476, 414, 369, 476, 476, 418, 476, 476, 421, 422, - /* 2540 */ 423, 424, 425, 426, 381, 428, 383, 414, 476, 476, - /* 2550 */ 476, 418, 476, 369, 421, 422, 423, 424, 425, 426, - /* 2560 */ 476, 428, 476, 333, 476, 381, 476, 383, 476, 476, - /* 2570 */ 476, 476, 476, 343, 476, 476, 476, 414, 476, 476, - /* 2580 */ 476, 418, 476, 476, 421, 422, 423, 424, 425, 426, - /* 2590 */ 476, 428, 476, 476, 476, 476, 476, 476, 414, 369, - /* 2600 */ 476, 476, 418, 476, 476, 421, 422, 423, 424, 425, - /* 2610 */ 426, 381, 428, 383, 476, 476, 476, 476, 476, 476, - /* 2620 */ 476, 476, 476, 333, 476, 476, 476, 476, 476, 476, - /* 2630 */ 476, 476, 476, 343, 476, 476, 333, 476, 476, 476, - /* 2640 */ 476, 476, 476, 476, 414, 476, 343, 476, 418, 476, - /* 2650 */ 476, 421, 422, 423, 424, 425, 426, 476, 428, 369, - /* 2660 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2670 */ 476, 381, 369, 383, 476, 476, 476, 476, 476, 476, - /* 2680 */ 476, 476, 476, 476, 381, 476, 383, 333, 476, 476, - /* 2690 */ 476, 476, 476, 476, 476, 476, 476, 343, 476, 476, - /* 2700 */ 476, 476, 476, 476, 414, 476, 476, 476, 418, 476, - /* 2710 */ 476, 421, 422, 423, 424, 425, 426, 414, 428, 476, - /* 2720 */ 476, 418, 476, 369, 421, 422, 423, 424, 425, 426, - /* 2730 */ 476, 428, 476, 333, 476, 381, 476, 383, 476, 476, - /* 2740 */ 476, 476, 476, 343, 476, 476, 476, 476, 333, 476, - /* 2750 */ 476, 476, 476, 476, 476, 476, 476, 476, 343, 476, - /* 2760 */ 476, 476, 476, 476, 476, 476, 476, 476, 414, 369, - /* 2770 */ 476, 476, 418, 476, 476, 421, 422, 423, 424, 425, - /* 2780 */ 426, 381, 428, 383, 369, 476, 476, 476, 476, 476, - /* 2790 */ 476, 476, 476, 476, 333, 476, 381, 476, 383, 476, - /* 2800 */ 476, 476, 476, 476, 343, 476, 476, 476, 476, 333, - /* 2810 */ 476, 476, 476, 476, 414, 476, 476, 476, 418, 343, - /* 2820 */ 476, 421, 422, 423, 424, 425, 426, 476, 428, 414, - /* 2830 */ 369, 476, 476, 418, 476, 476, 421, 422, 423, 424, - /* 2840 */ 425, 426, 381, 428, 383, 369, 476, 476, 476, 476, - /* 2850 */ 476, 476, 476, 476, 476, 476, 476, 381, 476, 383, + /* 1350 */ 229, 230, 231, 232, 233, 234, 235, 333, 12, 13, + /* 1360 */ 35, 101, 44, 101, 44, 44, 0, 343, 22, 345, + /* 1370 */ 44, 343, 44, 341, 101, 381, 392, 101, 101, 33, + /* 1380 */ 44, 35, 449, 333, 101, 101, 467, 44, 441, 451, + /* 1390 */ 254, 417, 416, 343, 370, 345, 49, 72, 184, 403, + /* 1400 */ 42, 389, 20, 392, 165, 59, 382, 389, 384, 387, + /* 1410 */ 20, 98, 342, 342, 387, 389, 50, 354, 72, 101, + /* 1420 */ 370, 101, 101, 215, 387, 96, 353, 101, 276, 101, + /* 1430 */ 333, 342, 382, 95, 384, 352, 342, 101, 342, 415, + /* 1440 */ 343, 342, 418, 97, 101, 421, 422, 423, 424, 425, + /* 1450 */ 426, 333, 428, 188, 188, 20, 48, 433, 335, 435, + /* 1460 */ 335, 343, 20, 439, 440, 415, 339, 370, 418, 339, + /* 1470 */ 410, 421, 422, 423, 424, 425, 426, 20, 428, 382, + /* 1480 */ 20, 384, 350, 433, 384, 435, 344, 350, 370, 439, + /* 1490 */ 440, 20, 402, 53, 350, 344, 350, 350, 350, 342, + /* 1500 */ 382, 347, 384, 350, 335, 347, 342, 335, 370, 370, + /* 1510 */ 370, 370, 415, 382, 203, 418, 382, 382, 421, 422, + /* 1520 */ 423, 424, 425, 426, 414, 428, 370, 100, 412, 410, + /* 1530 */ 370, 370, 186, 415, 188, 348, 418, 370, 370, 421, + /* 1540 */ 422, 423, 424, 425, 426, 370, 428, 370, 192, 348, + /* 1550 */ 191, 433, 190, 435, 409, 333, 382, 439, 440, 261, + /* 1560 */ 384, 215, 216, 408, 456, 343, 342, 260, 397, 407, + /* 1570 */ 473, 392, 392, 382, 228, 229, 230, 231, 232, 233, + /* 1580 */ 234, 397, 456, 269, 177, 382, 382, 459, 271, 270, + /* 1590 */ 255, 458, 370, 278, 343, 417, 251, 475, 275, 20, + /* 1600 */ 273, 420, 344, 342, 382, 455, 384, 20, 395, 397, + /* 1610 */ 382, 348, 348, 397, 382, 170, 348, 382, 382, 382, + /* 1620 */ 394, 333, 456, 382, 454, 343, 453, 348, 100, 100, + /* 1630 */ 438, 343, 382, 342, 360, 366, 36, 415, 348, 336, + /* 1640 */ 418, 335, 398, 421, 422, 423, 424, 425, 426, 333, + /* 1650 */ 428, 469, 374, 468, 411, 433, 404, 435, 370, 343, + /* 1660 */ 364, 439, 440, 364, 364, 331, 0, 0, 398, 0, + /* 1670 */ 382, 349, 384, 42, 0, 35, 208, 35, 35, 35, + /* 1680 */ 208, 333, 0, 35, 35, 208, 370, 0, 208, 0, + /* 1690 */ 35, 343, 22, 0, 35, 195, 188, 186, 382, 0, + /* 1700 */ 384, 0, 0, 415, 0, 182, 418, 181, 0, 421, + /* 1710 */ 422, 423, 424, 425, 426, 0, 428, 0, 370, 47, + /* 1720 */ 0, 433, 0, 435, 0, 42, 0, 439, 440, 0, + /* 1730 */ 382, 415, 384, 0, 418, 0, 0, 421, 422, 423, + /* 1740 */ 424, 425, 426, 0, 428, 0, 155, 35, 0, 155, + /* 1750 */ 0, 435, 333, 0, 0, 439, 440, 139, 42, 0, + /* 1760 */ 0, 0, 343, 415, 0, 0, 418, 0, 0, 421, + /* 1770 */ 422, 423, 424, 425, 426, 0, 428, 0, 0, 0, + /* 1780 */ 333, 0, 0, 435, 0, 0, 0, 439, 440, 370, + /* 1790 */ 343, 0, 0, 0, 22, 0, 0, 0, 0, 333, + /* 1800 */ 0, 382, 22, 384, 48, 22, 48, 0, 59, 343, + /* 1810 */ 0, 35, 0, 0, 59, 333, 0, 370, 44, 14, + /* 1820 */ 47, 59, 0, 0, 0, 343, 39, 0, 39, 382, + /* 1830 */ 42, 384, 0, 39, 415, 177, 370, 418, 0, 0, + /* 1840 */ 421, 422, 423, 424, 425, 426, 47, 428, 382, 47, + /* 1850 */ 384, 40, 370, 0, 435, 0, 66, 375, 439, 440, + /* 1860 */ 35, 0, 415, 49, 382, 418, 384, 39, 421, 422, + /* 1870 */ 423, 424, 425, 426, 39, 428, 333, 35, 49, 0, + /* 1880 */ 35, 415, 49, 39, 418, 0, 343, 421, 422, 423, + /* 1890 */ 424, 425, 426, 35, 428, 49, 430, 415, 1, 333, + /* 1900 */ 418, 39, 0, 421, 422, 423, 424, 425, 426, 343, + /* 1910 */ 428, 464, 0, 370, 0, 0, 19, 109, 375, 35, + /* 1920 */ 22, 0, 107, 35, 35, 382, 35, 384, 35, 35, + /* 1930 */ 33, 35, 22, 0, 35, 22, 370, 51, 35, 44, + /* 1940 */ 35, 375, 35, 0, 44, 22, 49, 0, 382, 22, + /* 1950 */ 384, 35, 55, 56, 57, 58, 59, 35, 415, 0, + /* 1960 */ 0, 418, 0, 35, 421, 422, 423, 424, 425, 426, + /* 1970 */ 22, 428, 20, 35, 35, 35, 101, 100, 0, 35, + /* 1980 */ 100, 415, 0, 22, 418, 168, 22, 421, 422, 423, + /* 1990 */ 424, 425, 426, 333, 428, 189, 99, 0, 168, 102, + /* 2000 */ 0, 170, 193, 343, 168, 3, 100, 44, 256, 175, + /* 2010 */ 101, 100, 44, 48, 48, 98, 96, 44, 44, 101, + /* 2020 */ 101, 44, 47, 47, 100, 333, 100, 256, 3, 100, + /* 2030 */ 370, 134, 101, 101, 100, 343, 101, 44, 35, 47, + /* 2040 */ 35, 35, 382, 333, 384, 35, 35, 35, 47, 0, + /* 2050 */ 101, 101, 44, 343, 0, 0, 0, 256, 100, 39, + /* 2060 */ 47, 171, 370, 0, 110, 39, 169, 100, 250, 47, + /* 2070 */ 101, 174, 101, 169, 382, 415, 384, 44, 418, 100, + /* 2080 */ 370, 421, 422, 423, 424, 425, 426, 100, 428, 100, + /* 2090 */ 333, 194, 382, 237, 384, 100, 98, 98, 2, 22, + /* 2100 */ 343, 22, 215, 35, 47, 47, 35, 415, 101, 100, + /* 2110 */ 418, 35, 122, 421, 422, 423, 424, 425, 426, 100, + /* 2120 */ 428, 35, 101, 100, 100, 415, 111, 370, 418, 101, + /* 2130 */ 100, 421, 422, 423, 424, 425, 426, 333, 428, 382, + /* 2140 */ 101, 384, 217, 100, 35, 101, 100, 343, 101, 101, + /* 2150 */ 100, 100, 35, 101, 101, 44, 122, 122, 122, 333, + /* 2160 */ 100, 100, 100, 35, 100, 100, 22, 66, 65, 343, + /* 2170 */ 35, 35, 415, 44, 370, 418, 35, 35, 421, 422, + /* 2180 */ 423, 424, 425, 426, 333, 428, 382, 35, 384, 35, + /* 2190 */ 35, 35, 72, 94, 343, 35, 370, 35, 35, 22, + /* 2200 */ 35, 35, 35, 72, 35, 35, 333, 35, 382, 35, + /* 2210 */ 384, 35, 22, 35, 0, 35, 343, 49, 39, 415, + /* 2220 */ 0, 370, 418, 35, 49, 421, 422, 423, 424, 425, + /* 2230 */ 426, 39, 428, 382, 0, 384, 35, 39, 49, 0, + /* 2240 */ 35, 415, 49, 370, 418, 39, 0, 421, 422, 423, + /* 2250 */ 424, 425, 426, 35, 428, 382, 35, 384, 22, 21, + /* 2260 */ 476, 476, 22, 22, 21, 20, 415, 0, 476, 418, + /* 2270 */ 333, 476, 421, 422, 423, 424, 425, 426, 476, 428, + /* 2280 */ 343, 476, 476, 476, 476, 476, 476, 476, 415, 476, + /* 2290 */ 333, 418, 476, 476, 421, 422, 423, 424, 425, 426, + /* 2300 */ 343, 428, 476, 476, 476, 333, 476, 370, 476, 476, + /* 2310 */ 476, 476, 476, 476, 476, 343, 476, 476, 476, 382, + /* 2320 */ 476, 384, 476, 476, 476, 476, 476, 370, 476, 476, + /* 2330 */ 476, 476, 476, 476, 476, 476, 476, 333, 476, 382, + /* 2340 */ 476, 384, 370, 476, 476, 476, 476, 343, 476, 476, + /* 2350 */ 476, 476, 415, 476, 382, 418, 384, 476, 421, 422, + /* 2360 */ 423, 424, 425, 426, 476, 428, 476, 476, 476, 476, + /* 2370 */ 476, 476, 415, 476, 370, 418, 476, 476, 421, 422, + /* 2380 */ 423, 424, 425, 426, 476, 428, 382, 415, 384, 476, + /* 2390 */ 418, 476, 476, 421, 422, 423, 424, 425, 426, 476, + /* 2400 */ 428, 476, 476, 476, 476, 333, 476, 476, 476, 476, + /* 2410 */ 476, 476, 476, 476, 476, 343, 476, 476, 476, 415, + /* 2420 */ 476, 476, 418, 476, 333, 421, 422, 423, 424, 425, + /* 2430 */ 426, 476, 428, 476, 343, 476, 476, 476, 476, 476, + /* 2440 */ 476, 476, 370, 476, 476, 476, 476, 476, 476, 476, + /* 2450 */ 476, 476, 333, 476, 382, 476, 384, 476, 476, 476, + /* 2460 */ 476, 370, 343, 476, 476, 476, 476, 476, 476, 476, + /* 2470 */ 476, 333, 476, 382, 476, 384, 476, 476, 476, 476, + /* 2480 */ 476, 343, 476, 476, 476, 476, 476, 415, 476, 370, + /* 2490 */ 418, 476, 476, 421, 422, 423, 424, 425, 426, 476, + /* 2500 */ 428, 382, 476, 384, 476, 476, 415, 476, 370, 418, + /* 2510 */ 476, 476, 421, 422, 423, 424, 425, 426, 333, 428, + /* 2520 */ 382, 476, 384, 476, 476, 476, 476, 476, 343, 476, + /* 2530 */ 476, 476, 476, 476, 415, 476, 476, 418, 476, 476, + /* 2540 */ 421, 422, 423, 424, 425, 426, 476, 428, 476, 476, + /* 2550 */ 476, 476, 476, 415, 476, 370, 418, 476, 476, 421, + /* 2560 */ 422, 423, 424, 425, 426, 476, 428, 382, 476, 384, + /* 2570 */ 476, 476, 476, 476, 476, 333, 476, 476, 476, 476, + /* 2580 */ 476, 476, 476, 476, 476, 343, 476, 476, 476, 476, + /* 2590 */ 476, 333, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2600 */ 415, 343, 476, 418, 476, 476, 421, 422, 423, 424, + /* 2610 */ 425, 426, 370, 428, 476, 476, 476, 476, 476, 476, + /* 2620 */ 476, 476, 476, 476, 382, 476, 384, 476, 370, 476, + /* 2630 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2640 */ 382, 333, 384, 476, 476, 476, 476, 476, 476, 476, + /* 2650 */ 476, 343, 476, 476, 476, 476, 476, 415, 476, 476, + /* 2660 */ 418, 333, 476, 421, 422, 423, 424, 425, 426, 476, + /* 2670 */ 428, 343, 476, 415, 476, 476, 418, 476, 370, 421, + /* 2680 */ 422, 423, 424, 425, 426, 476, 428, 476, 476, 476, + /* 2690 */ 382, 476, 384, 476, 476, 476, 476, 476, 370, 476, + /* 2700 */ 476, 476, 476, 476, 476, 476, 476, 476, 333, 476, + /* 2710 */ 382, 476, 384, 476, 476, 476, 476, 476, 343, 476, + /* 2720 */ 476, 476, 476, 415, 476, 476, 418, 476, 476, 421, + /* 2730 */ 422, 423, 424, 425, 426, 476, 428, 476, 476, 476, + /* 2740 */ 476, 476, 33, 415, 476, 370, 418, 476, 476, 421, + /* 2750 */ 422, 423, 424, 425, 426, 476, 428, 382, 49, 384, + /* 2760 */ 476, 476, 476, 476, 55, 56, 57, 58, 59, 476, + /* 2770 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2780 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2790 */ 415, 476, 476, 418, 476, 476, 421, 422, 423, 424, + /* 2800 */ 425, 426, 476, 428, 476, 476, 476, 476, 99, 476, + /* 2810 */ 476, 102, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2820 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2830 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2840 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2850 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, /* 2860 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2870 */ 476, 476, 476, 476, 476, 414, 476, 476, 476, 418, - /* 2880 */ 476, 476, 421, 422, 423, 424, 425, 426, 476, 428, - /* 2890 */ 414, 476, 476, 476, 418, 476, 476, 421, 422, 423, - /* 2900 */ 424, 425, 426, 476, 428, + /* 2870 */ 476, 476, 476, 476, 476, 476, 476, 476, 169, 170, + /* 2880 */ 476, 476, 476, 174, 476, 176, 476, 476, 476, 476, + /* 2890 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2900 */ 476, 476, 476, 194, 476, 476, 476, 476, 476, 476, + /* 2910 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2920 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2930 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2940 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2950 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2960 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2970 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2980 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2990 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 3000 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 3010 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 3020 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 3030 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 3040 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 3050 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 3060 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 3070 */ 476, 476, 476, 330, 330, 330, 330, 330, 330, 330, + /* 3080 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3090 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3100 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3110 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3120 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3130 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3140 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3150 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3160 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3170 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3180 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3190 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3200 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3210 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3220 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3230 */ 330, 330, 330, 330, }; #define YY_SHIFT_COUNT (761) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (2182) +#define YY_SHIFT_MAX (2709) static const unsigned short int yy_shift_ofst[] = { /* 0 */ 550, 0, 224, 0, 449, 449, 449, 449, 449, 449, /* 10 */ 449, 449, 449, 449, 449, 449, 673, 897, 897, 1121, /* 20 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 897, /* 30 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 897, - /* 40 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 457, - /* 50 */ 492, 142, 145, 147, 107, 173, 107, 145, 145, 1347, - /* 60 */ 1347, 1347, 107, 1347, 1347, 501, 107, 18, 447, 47, - /* 70 */ 47, 447, 235, 235, 3, 286, 7, 7, 47, 47, - /* 80 */ 47, 47, 47, 47, 47, 51, 47, 47, 132, 18, - /* 90 */ 47, 47, 228, 47, 18, 47, 51, 47, 51, 18, - /* 100 */ 47, 47, 18, 47, 18, 18, 18, 47, 339, 223, - /* 110 */ 189, 189, 333, 114, 443, 443, 443, 443, 443, 443, + /* 40 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 148, + /* 50 */ 368, 93, 64, 140, 147, 322, 147, 64, 64, 1346, + /* 60 */ 1346, 1346, 147, 1346, 1346, 314, 147, 76, 369, 111, + /* 70 */ 111, 369, 656, 656, 18, 327, 7, 7, 111, 111, + /* 80 */ 111, 111, 111, 111, 111, 145, 111, 111, 248, 76, + /* 90 */ 111, 111, 363, 111, 76, 111, 145, 111, 145, 76, + /* 100 */ 111, 111, 76, 111, 76, 76, 76, 111, 343, 223, + /* 110 */ 189, 189, 405, 114, 443, 443, 443, 443, 443, 443, /* 120 */ 443, 443, 443, 443, 443, 443, 443, 443, 443, 443, - /* 130 */ 443, 443, 443, 843, 325, 3, 286, 392, 220, 220, - /* 140 */ 220, 417, 369, 369, 392, 344, 344, 344, 132, 365, - /* 150 */ 297, 18, 439, 18, 439, 439, 503, 558, 35, 35, - /* 160 */ 35, 35, 35, 35, 35, 35, 2060, 217, 288, 483, - /* 170 */ 522, 260, 313, 243, 384, 940, 576, 484, 890, 751, - /* 180 */ 528, 810, 824, 60, 974, 824, 1065, 791, 275, 1124, - /* 190 */ 1336, 1204, 1353, 1378, 1353, 1236, 1384, 1384, 1353, 1236, - /* 200 */ 1236, 1315, 1324, 1384, 1329, 1384, 1384, 1384, 1411, 1386, - /* 210 */ 1411, 1386, 1418, 132, 1421, 132, 1426, 1430, 132, 1426, - /* 220 */ 132, 132, 132, 1384, 132, 1413, 1413, 1411, 18, 18, - /* 230 */ 18, 18, 18, 18, 18, 18, 18, 18, 18, 1384, - /* 240 */ 1411, 439, 439, 439, 1292, 1398, 1418, 339, 1325, 1323, - /* 250 */ 1421, 339, 1330, 1384, 1378, 1378, 439, 1262, 1266, 439, - /* 260 */ 1262, 1266, 439, 439, 18, 1253, 1355, 1262, 1271, 1273, - /* 270 */ 1293, 1124, 1283, 1277, 1276, 1318, 344, 1550, 1384, 1426, - /* 280 */ 339, 339, 1566, 1266, 439, 439, 439, 439, 439, 1266, - /* 290 */ 439, 1427, 339, 503, 339, 344, 1504, 1508, 439, 558, - /* 300 */ 1384, 339, 1576, 1411, 2905, 2905, 2905, 2905, 2905, 2905, - /* 310 */ 2905, 2905, 2905, 34, 430, 15, 590, 734, 16, 839, - /* 320 */ 79, 709, 792, 566, 856, 1023, 1023, 1023, 1023, 1023, - /* 330 */ 1023, 1023, 1023, 1023, 1167, 743, 253, 253, 402, 57, - /* 340 */ 758, 774, 949, 1067, 800, 980, 64, 64, 959, 1007, - /* 350 */ 387, 959, 959, 959, 1149, 641, 1116, 1140, 1147, 1070, - /* 360 */ 1168, 1078, 1119, 1141, 1164, 532, 1190, 1207, 1250, 1251, - /* 370 */ 990, 1189, 1212, 868, 1216, 1231, 1233, 943, 1004, 1117, - /* 380 */ 1153, 1261, 1264, 1267, 1270, 1272, 1274, 1302, 1289, 1196, - /* 390 */ 1206, 987, 1291, 1238, 1286, 1317, 1320, 1322, 1326, 1328, - /* 400 */ 190, 1248, 1361, 1256, 1331, 1614, 1620, 1624, 1590, 1633, - /* 410 */ 1599, 1428, 1600, 1602, 1604, 1432, 1644, 1607, 1610, 1438, - /* 420 */ 1647, 1440, 1649, 1615, 1651, 1630, 1653, 1621, 1460, 1469, - /* 430 */ 1473, 1660, 1667, 1669, 1488, 1490, 1672, 1689, 1629, 1690, - /* 440 */ 1691, 1692, 1652, 1696, 1697, 1698, 1702, 1703, 1704, 1705, - /* 450 */ 1706, 1552, 1673, 1709, 1555, 1712, 1714, 1715, 1716, 1717, - /* 460 */ 1718, 1719, 1720, 1721, 1722, 1723, 1724, 1726, 1728, 1729, - /* 470 */ 1730, 1693, 1731, 1732, 1733, 1734, 1736, 1738, 1725, 1739, - /* 480 */ 1740, 1741, 1603, 1743, 1744, 1735, 1700, 1737, 1707, 1745, - /* 490 */ 1687, 1742, 1750, 1694, 1752, 1695, 1756, 1760, 1753, 1746, - /* 500 */ 1727, 1751, 1755, 1747, 1757, 1773, 1759, 1748, 1774, 1775, - /* 510 */ 1776, 1758, 1586, 1764, 1780, 1781, 1749, 1800, 1805, 1771, - /* 520 */ 1761, 1770, 1812, 1778, 1767, 1779, 1817, 1784, 1772, 1783, - /* 530 */ 1820, 1788, 1782, 1786, 1827, 1829, 1830, 1832, 1766, 1754, - /* 540 */ 1798, 1813, 1836, 1803, 1804, 1806, 1807, 1808, 1809, 1810, - /* 550 */ 1796, 1802, 1815, 1818, 1826, 1819, 1852, 1833, 1856, 1840, - /* 560 */ 1814, 1858, 1842, 1841, 1877, 1843, 1879, 1846, 1887, 1866, - /* 570 */ 1878, 1864, 1865, 1867, 1821, 1801, 1904, 1762, 1811, 1713, - /* 580 */ 1872, 1896, 1921, 1765, 1901, 1777, 1789, 1924, 1925, 1785, - /* 590 */ 1768, 1923, 1883, 1675, 1834, 1828, 1844, 1885, 1849, 1893, - /* 600 */ 1853, 1847, 1906, 1908, 1855, 1857, 1861, 1862, 1863, 1919, - /* 610 */ 1918, 1922, 1868, 1926, 1710, 1870, 1871, 1964, 1929, 1791, - /* 620 */ 1939, 1941, 1942, 1943, 1944, 1945, 1880, 1881, 1937, 1790, - /* 630 */ 1946, 1947, 1983, 1986, 1988, 1989, 1892, 1954, 1751, 1950, - /* 640 */ 1907, 1903, 1909, 1912, 1913, 1837, 1914, 2015, 1977, 1848, - /* 650 */ 1927, 1910, 1751, 1971, 1978, 1928, 1792, 1930, 2019, 2002, - /* 660 */ 1816, 1933, 1934, 1938, 1935, 1948, 1940, 1987, 1949, 1951, - /* 670 */ 1992, 1952, 2008, 1825, 1955, 1932, 1953, 2017, 2023, 1962, - /* 680 */ 1968, 2036, 1973, 1974, 2039, 1976, 1979, 2043, 1982, 1991, - /* 690 */ 2048, 1995, 1981, 1985, 1990, 1999, 2005, 2064, 2010, 2076, - /* 700 */ 2013, 2064, 2064, 2092, 2056, 2058, 2089, 2090, 2091, 2093, - /* 710 */ 2094, 2096, 2098, 2099, 2100, 2101, 2055, 2044, 2095, 2102, - /* 720 */ 2105, 2106, 2120, 2108, 2110, 2112, 2077, 1796, 2113, 1802, - /* 730 */ 2115, 2116, 2118, 2119, 2130, 2122, 2155, 2123, 2111, 2124, - /* 740 */ 2161, 2129, 2125, 2126, 2166, 2132, 2127, 2134, 2169, 2136, - /* 750 */ 2131, 2139, 2172, 2140, 2144, 2182, 2168, 2171, 2173, 2174, - /* 760 */ 2176, 2178, + /* 130 */ 443, 443, 443, 672, 483, 18, 327, 293, 253, 253, + /* 140 */ 253, 217, 622, 622, 293, 523, 523, 523, 248, 460, + /* 150 */ 354, 76, 585, 76, 585, 585, 604, 670, 35, 35, + /* 160 */ 35, 35, 35, 35, 35, 35, 1897, 636, 15, 167, + /* 170 */ 16, 297, 515, 252, 454, 454, 373, 842, 389, 484, + /* 180 */ 1058, 1163, 862, 892, 922, 983, 45, 922, 878, 727, + /* 190 */ 948, 1136, 1347, 1214, 1358, 1382, 1358, 1239, 1390, 1390, + /* 200 */ 1358, 1239, 1239, 1313, 1329, 1390, 1338, 1390, 1390, 1390, + /* 210 */ 1435, 1408, 1435, 1408, 1442, 248, 1457, 248, 1460, 1471, + /* 220 */ 248, 1460, 248, 248, 248, 1390, 248, 1440, 1440, 1435, + /* 230 */ 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, + /* 240 */ 76, 1390, 1435, 585, 585, 585, 1311, 1427, 1442, 343, + /* 250 */ 1356, 1359, 1457, 343, 1362, 1390, 1382, 1382, 585, 1298, + /* 260 */ 1307, 585, 1298, 1307, 585, 585, 76, 1314, 1407, 1298, + /* 270 */ 1317, 1319, 1335, 1136, 1315, 1323, 1327, 1345, 523, 1579, + /* 280 */ 1390, 1460, 343, 343, 1587, 1307, 585, 585, 585, 585, + /* 290 */ 585, 1307, 585, 1445, 343, 604, 343, 523, 1528, 1529, + /* 300 */ 585, 670, 1390, 343, 1600, 1435, 2904, 2904, 2904, 2904, + /* 310 */ 2904, 2904, 2904, 2904, 2904, 34, 2709, 238, 590, 198, + /* 320 */ 539, 973, 794, 486, 1064, 925, 1030, 48, 48, 48, + /* 330 */ 48, 48, 48, 48, 48, 48, 831, 612, 65, 65, + /* 340 */ 549, 737, 700, 891, 262, 448, 1025, 717, 531, 531, + /* 350 */ 962, 751, 906, 962, 962, 962, 1202, 1015, 513, 1219, + /* 360 */ 1008, 1101, 1194, 1104, 1113, 1149, 1159, 381, 1229, 1248, + /* 370 */ 1259, 1267, 1069, 1196, 1221, 1217, 1231, 1233, 1234, 1143, + /* 380 */ 836, 1152, 1165, 1260, 1262, 1273, 1276, 1277, 1318, 1281, + /* 390 */ 1283, 1208, 1284, 1238, 1320, 1321, 1326, 1328, 1336, 1343, + /* 400 */ 671, 1265, 1266, 1255, 1289, 1325, 1366, 1666, 1667, 1669, + /* 410 */ 1631, 1674, 1640, 1468, 1642, 1643, 1644, 1472, 1682, 1648, + /* 420 */ 1649, 1477, 1687, 1480, 1689, 1655, 1699, 1670, 1693, 1659, + /* 430 */ 1500, 1508, 1511, 1701, 1702, 1704, 1523, 1526, 1708, 1715, + /* 440 */ 1672, 1717, 1720, 1722, 1683, 1724, 1726, 1729, 1733, 1735, + /* 450 */ 1736, 1743, 1745, 1591, 1712, 1748, 1594, 1750, 1753, 1754, + /* 460 */ 1764, 1765, 1767, 1768, 1775, 1777, 1778, 1779, 1781, 1782, + /* 470 */ 1784, 1785, 1786, 1716, 1759, 1760, 1761, 1791, 1792, 1793, + /* 480 */ 1772, 1795, 1796, 1797, 1618, 1798, 1800, 1780, 1756, 1783, + /* 490 */ 1758, 1807, 1749, 1776, 1810, 1755, 1812, 1762, 1813, 1816, + /* 500 */ 1788, 1787, 1774, 1773, 1799, 1805, 1802, 1822, 1811, 1789, + /* 510 */ 1823, 1824, 1827, 1794, 1658, 1832, 1838, 1839, 1790, 1853, + /* 520 */ 1855, 1825, 1814, 1828, 1861, 1842, 1829, 1835, 1879, 1845, + /* 530 */ 1833, 1844, 1885, 1858, 1846, 1862, 1902, 1912, 1914, 1915, + /* 540 */ 1808, 1815, 1884, 1898, 1921, 1888, 1889, 1891, 1893, 1894, + /* 550 */ 1896, 1899, 1895, 1900, 1903, 1905, 1910, 1907, 1933, 1913, + /* 560 */ 1943, 1923, 1886, 1947, 1927, 1916, 1959, 1922, 1960, 1928, + /* 570 */ 1962, 1948, 1952, 1938, 1939, 1940, 1875, 1877, 1978, 1817, + /* 580 */ 1880, 1809, 1944, 1961, 1982, 1806, 1964, 1830, 1831, 1997, + /* 590 */ 2000, 1836, 1834, 2002, 1963, 1752, 1906, 1909, 1911, 1965, + /* 600 */ 1917, 1966, 1920, 1918, 1968, 1973, 1919, 1924, 1926, 1929, + /* 610 */ 1931, 1974, 1975, 1976, 1934, 1977, 1771, 1932, 1935, 2025, + /* 620 */ 1993, 1801, 2003, 2005, 2006, 2010, 2011, 2012, 1949, 1950, + /* 630 */ 1992, 1818, 2008, 2001, 2049, 2054, 2055, 2056, 1958, 2020, + /* 640 */ 1773, 2013, 1967, 1969, 1971, 1979, 1987, 1890, 1989, 2063, + /* 650 */ 2026, 1904, 1995, 1954, 1773, 2022, 2033, 1998, 1856, 1999, + /* 660 */ 2096, 2077, 1887, 2009, 2007, 2019, 2021, 2023, 2028, 2057, + /* 670 */ 2024, 2030, 2058, 2039, 2079, 1925, 2043, 2015, 2044, 2068, + /* 680 */ 2071, 2046, 2047, 2076, 2050, 2048, 2086, 2051, 2052, 2109, + /* 690 */ 2060, 2053, 2117, 2061, 1990, 2034, 2035, 2036, 2062, 2111, + /* 700 */ 2064, 2128, 2065, 2111, 2111, 2144, 2101, 2103, 2135, 2136, + /* 710 */ 2141, 2142, 2152, 2154, 2155, 2156, 2120, 2099, 2129, 2160, + /* 720 */ 2162, 2163, 2177, 2165, 2166, 2167, 2131, 1895, 2169, 1900, + /* 730 */ 2170, 2172, 2174, 2176, 2190, 2178, 2214, 2180, 2168, 2179, + /* 740 */ 2220, 2188, 2175, 2192, 2234, 2201, 2189, 2198, 2239, 2205, + /* 750 */ 2193, 2206, 2246, 2218, 2221, 2267, 2236, 2238, 2240, 2241, + /* 760 */ 2243, 2245, }; -#define YY_REDUCE_COUNT (312) -#define YY_REDUCE_MIN (-396) -#define YY_REDUCE_MAX (2476) +#define YY_REDUCE_COUNT (314) +#define YY_REDUCE_MIN (-397) +#define YY_REDUCE_MAX (2375) static const short yy_reduce_ofst[] = { - /* 0 */ -140, 137, -82, 358, 444, 578, 814, 841, 1034, 1064, - /* 10 */ 1132, 1240, 1260, 1344, 1368, 1445, 598, -333, 672, 1468, - /* 20 */ 901, 1491, 1514, 1577, 1642, 1663, 1676, 1763, 1787, 1850, - /* 30 */ 1874, 1920, 1936, 1996, 2020, 2047, 2066, 2117, 2133, 2163, - /* 40 */ 2184, 2230, 2290, 2303, 2354, 2400, 2415, 2461, 2476, 372, - /* 50 */ -195, -396, -265, 487, 595, 607, 701, 481, 761, -362, - /* 60 */ -355, 285, 749, 136, 612, -279, -55, 256, -346, -167, - /* 70 */ -5, -381, -334, -257, -205, -249, 45, 130, 300, 303, - /* 80 */ 356, 377, 409, 425, 432, 177, 469, 489, 252, -308, - /* 90 */ 521, 573, 170, 620, 172, 622, 261, 707, 396, 682, - /* 100 */ 739, 745, 278, 793, 715, 321, 781, 795, -266, 8, - /* 110 */ -350, -350, 115, -238, 161, 214, 257, 315, 374, 433, - /* 120 */ 445, 463, 585, 587, 617, 677, 681, 806, 807, 813, - /* 130 */ 819, 830, 832, -86, -289, -111, 23, 154, -289, 70, - /* 140 */ 188, -25, 480, 511, 380, 416, 498, 704, 381, -354, - /* 150 */ 426, 513, 690, 502, 733, 798, 808, 353, -369, -365, - /* 160 */ 324, 370, 458, 505, 686, 458, 764, 368, 621, 825, - /* 170 */ 753, 780, 906, 822, 917, 917, 946, 898, 954, 963, - /* 180 */ 930, 921, 871, 871, 891, 871, 916, 925, 917, 967, - /* 190 */ 971, 989, 1001, 1008, 1012, 1016, 1063, 1066, 1022, 1025, - /* 200 */ 1026, 1060, 1069, 1074, 1073, 1084, 1086, 1088, 1097, 1096, - /* 210 */ 1101, 1098, 1030, 1090, 1059, 1093, 1105, 1050, 1103, 1110, - /* 220 */ 1111, 1113, 1114, 1126, 1120, 1125, 1129, 1136, 1108, 1112, - /* 230 */ 1115, 1122, 1127, 1131, 1133, 1137, 1138, 1139, 1142, 1151, - /* 240 */ 1144, 1099, 1102, 1128, 1081, 1094, 1107, 1162, 1104, 1118, - /* 250 */ 1135, 1171, 1123, 1179, 1143, 1145, 1146, 1068, 1134, 1150, - /* 260 */ 1072, 1148, 1156, 1157, 917, 1080, 1082, 1085, 1092, 1091, - /* 270 */ 1106, 1152, 1076, 1095, 1130, 871, 1223, 1154, 1242, 1241, - /* 280 */ 1234, 1239, 1194, 1193, 1209, 1210, 1211, 1213, 1214, 1200, - /* 290 */ 1218, 1208, 1252, 1237, 1257, 1263, 1169, 1243, 1229, 1254, - /* 300 */ 1269, 1278, 1279, 1282, 1215, 1217, 1222, 1225, 1265, 1268, - /* 310 */ 1275, 1281, 1294, + /* 0 */ -156, 364, -82, 588, -20, 614, 804, 890, 1024, 1050, + /* 10 */ 1118, 1222, 1288, 1316, 1348, 1419, 174, -333, 154, 450, + /* 20 */ 674, 1447, 1097, 1482, 1466, 1543, 1566, 1660, 1692, 1710, + /* 30 */ 1757, 1804, 1826, 1851, 1873, 1937, 1957, 1972, 2004, 2072, + /* 40 */ 2091, 2119, 2138, 2185, 2242, 2258, 2308, 2328, 2375, 383, + /* 50 */ 270, -397, 83, -395, 371, 416, 554, 75, 501, 235, + /* 60 */ 786, 827, -139, 683, 834, -195, -165, -359, -80, -172, + /* 70 */ -13, -383, -254, -207, -213, 149, -52, 290, -152, -5, + /* 80 */ 263, 376, 407, 413, 453, 82, 508, 593, 462, 271, + /* 90 */ 610, 631, 275, 657, 374, 725, 301, 745, 372, -170, + /* 100 */ 748, 796, 399, 808, -83, 421, 605, 810, 260, -319, + /* 110 */ -338, -338, -293, -327, 94, 237, 433, 502, 534, 669, + /* 120 */ 693, 712, 802, 807, 809, 813, 815, 833, 849, 854, + /* 130 */ 865, 867, 868, -275, -251, -131, 355, 467, -251, 437, + /* 140 */ 559, 452, 445, 479, 726, -334, -85, 719, 681, 168, + /* 150 */ 736, 268, 134, 805, 797, 799, 742, 850, -369, 387, + /* 160 */ 499, 580, 713, 832, 913, 713, 637, 927, 485, 899, + /* 170 */ 818, 835, 949, 839, 941, 945, 940, 940, 980, 939, + /* 180 */ 1032, 1028, 994, 984, 933, 933, 919, 933, 947, 938, + /* 190 */ 940, 974, 976, 996, 1012, 1011, 1018, 1022, 1070, 1071, + /* 200 */ 1026, 1027, 1037, 1063, 1073, 1089, 1083, 1094, 1096, 1099, + /* 210 */ 1123, 1127, 1125, 1130, 1060, 1132, 1100, 1137, 1142, 1090, + /* 220 */ 1144, 1151, 1146, 1147, 1148, 1157, 1153, 1154, 1158, 1169, + /* 230 */ 1138, 1139, 1140, 1141, 1156, 1160, 1161, 1167, 1168, 1175, + /* 240 */ 1177, 1164, 1172, 1131, 1134, 1135, 1110, 1116, 1119, 1187, + /* 250 */ 1145, 1155, 1176, 1201, 1162, 1224, 1179, 1180, 1174, 1108, + /* 260 */ 1171, 1191, 1126, 1184, 1203, 1204, 940, 1128, 1133, 1166, + /* 270 */ 1150, 1170, 1173, 1178, 1122, 1182, 1185, 933, 1251, 1181, + /* 280 */ 1261, 1258, 1263, 1264, 1213, 1212, 1228, 1232, 1235, 1236, + /* 290 */ 1237, 1216, 1241, 1226, 1268, 1269, 1279, 1282, 1192, 1278, + /* 300 */ 1250, 1274, 1291, 1290, 1303, 1306, 1252, 1243, 1244, 1270, + /* 310 */ 1296, 1299, 1300, 1322, 1334, }; static const YYACTIONTYPE yy_default[] = { /* 0 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, @@ -941,28 +975,28 @@ static const YYACTIONTYPE yy_default[] = { /* 140 */ 2218, 1799, 2178, 2178, 1719, 1719, 1719, 1719, 1801, 2050, /* 150 */ 1719, 1719, 1719, 1719, 1719, 1719, 1920, 1719, 1719, 1719, /* 160 */ 1719, 1719, 1944, 1719, 1719, 1719, 2044, 1719, 1719, 2243, - /* 170 */ 2299, 1719, 1719, 2246, 1719, 1719, 1719, 1997, 1719, 1719, - /* 180 */ 1874, 2233, 2210, 2224, 2283, 2211, 2208, 2227, 1719, 2237, - /* 190 */ 1719, 2031, 1990, 1719, 1990, 1987, 1719, 1719, 1990, 1987, - /* 200 */ 1987, 1863, 1859, 1719, 1857, 1719, 1719, 1719, 1719, 1766, - /* 210 */ 1719, 1766, 1719, 1801, 1719, 1801, 1719, 1719, 1801, 1719, - /* 220 */ 1801, 1801, 1801, 1719, 1801, 1779, 1779, 1719, 1719, 1719, + /* 170 */ 2299, 1719, 1719, 2246, 1719, 1719, 1719, 1719, 1719, 1997, + /* 180 */ 1719, 1719, 1874, 2233, 2210, 2224, 2283, 2211, 2208, 2227, + /* 190 */ 1719, 2237, 1719, 2031, 1990, 1719, 1990, 1987, 1719, 1719, + /* 200 */ 1990, 1987, 1987, 1863, 1859, 1719, 1857, 1719, 1719, 1719, + /* 210 */ 1719, 1766, 1719, 1766, 1719, 1801, 1719, 1801, 1719, 1719, + /* 220 */ 1801, 1719, 1801, 1801, 1801, 1719, 1801, 1779, 1779, 1719, /* 230 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 240 */ 1719, 1719, 1719, 1719, 2062, 2048, 1719, 1799, 2042, 2040, - /* 250 */ 1719, 1799, 2038, 1719, 1719, 1719, 1719, 2254, 2252, 1719, - /* 260 */ 2254, 2252, 1719, 1719, 1719, 2268, 2264, 2254, 2272, 2270, - /* 270 */ 2239, 2237, 2302, 2289, 2285, 2224, 1719, 1719, 1719, 1719, - /* 280 */ 1799, 1799, 1719, 2252, 1719, 1719, 1719, 1719, 1719, 2252, - /* 290 */ 1719, 1719, 1799, 1719, 1799, 1719, 1719, 1890, 1719, 1719, - /* 300 */ 1719, 1799, 1751, 1719, 2033, 2053, 2015, 2015, 1923, 1923, - /* 310 */ 1923, 1802, 1724, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 320 */ 1719, 1719, 1719, 1719, 1719, 2267, 2266, 2133, 1719, 2182, - /* 330 */ 2181, 2180, 2171, 2132, 1886, 1719, 2131, 2130, 1719, 1719, - /* 340 */ 1719, 1719, 1719, 1719, 1719, 1719, 2006, 2005, 2124, 1719, - /* 350 */ 1719, 2125, 2123, 2122, 1719, 1719, 1719, 1719, 1719, 1719, + /* 240 */ 1719, 1719, 1719, 1719, 1719, 1719, 2062, 2048, 1719, 1799, + /* 250 */ 2042, 2040, 1719, 1799, 2038, 1719, 1719, 1719, 1719, 2254, + /* 260 */ 2252, 1719, 2254, 2252, 1719, 1719, 1719, 2268, 2264, 2254, + /* 270 */ 2272, 2270, 2239, 2237, 2302, 2289, 2285, 2224, 1719, 1719, + /* 280 */ 1719, 1719, 1799, 1799, 1719, 2252, 1719, 1719, 1719, 1719, + /* 290 */ 1719, 2252, 1719, 1719, 1799, 1719, 1799, 1719, 1719, 1890, + /* 300 */ 1719, 1719, 1719, 1799, 1751, 1719, 2033, 2053, 2015, 2015, + /* 310 */ 1923, 1923, 1923, 1802, 1724, 1719, 1719, 1719, 1719, 1719, + /* 320 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 2267, 2266, 2133, + /* 330 */ 1719, 2182, 2181, 2180, 2171, 2132, 1886, 1719, 2131, 2130, + /* 340 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 2006, 2005, + /* 350 */ 2124, 1719, 1719, 2125, 2123, 2122, 1719, 1719, 1719, 1719, /* 360 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 370 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 2286, 2290, - /* 380 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 2207, 1719, 1719, + /* 370 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 380 */ 2286, 2290, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 2207, /* 390 */ 1719, 1719, 1719, 2106, 1719, 1719, 1719, 1719, 1719, 1719, /* 400 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, /* 410 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, @@ -974,27 +1008,27 @@ static const YYACTIONTYPE yy_default[] = { /* 470 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, /* 480 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, /* 490 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 500 */ 1756, 2111, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 500 */ 1719, 1719, 1756, 2111, 1719, 1719, 1719, 1719, 1719, 1719, /* 510 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, /* 520 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, /* 530 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, /* 540 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 550 */ 1840, 1839, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 550 */ 1719, 1719, 1840, 1839, 1719, 1719, 1719, 1719, 1719, 1719, /* 560 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 570 */ 1719, 1719, 1719, 1719, 2115, 1719, 1719, 1719, 1719, 1719, + /* 570 */ 1719, 1719, 1719, 1719, 1719, 1719, 2115, 1719, 1719, 1719, /* 580 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 590 */ 1719, 2282, 2240, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 590 */ 1719, 1719, 1719, 2282, 2240, 1719, 1719, 1719, 1719, 1719, /* 600 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 610 */ 1719, 2106, 1719, 2265, 1719, 1719, 2280, 1719, 2284, 1719, - /* 620 */ 1719, 1719, 1719, 1719, 1719, 1719, 2217, 2213, 1719, 1719, - /* 630 */ 2209, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 2114, 1719, - /* 640 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 650 */ 1719, 1719, 2105, 1719, 2168, 1719, 1719, 1719, 2202, 1719, - /* 660 */ 1719, 2153, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 670 */ 1719, 2115, 1719, 2118, 1719, 1719, 1719, 1719, 1719, 1917, - /* 680 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 690 */ 1719, 1719, 1902, 1900, 1899, 1898, 1719, 1930, 1719, 1719, - /* 700 */ 1719, 1926, 1925, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 610 */ 1719, 1719, 1719, 2106, 1719, 2265, 1719, 1719, 2280, 1719, + /* 620 */ 2284, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 2217, 2213, + /* 630 */ 1719, 1719, 2209, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 640 */ 2114, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 650 */ 1719, 1719, 1719, 1719, 2105, 1719, 2168, 1719, 1719, 1719, + /* 660 */ 2202, 1719, 1719, 2153, 1719, 1719, 1719, 1719, 1719, 1719, + /* 670 */ 1719, 1719, 1719, 2115, 1719, 2118, 1719, 1719, 1719, 1719, + /* 680 */ 1719, 1917, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 690 */ 1719, 1719, 1719, 1719, 1902, 1900, 1899, 1898, 1719, 1930, + /* 700 */ 1719, 1719, 1719, 1926, 1925, 1719, 1719, 1719, 1719, 1719, /* 710 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1820, 1719, /* 720 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1812, 1719, 1811, /* 730 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, @@ -1795,66 +1829,66 @@ static const char *const yyTokenName[] = { /* 355 */ "integer_list", /* 356 */ "variable_list", /* 357 */ "retention_list", - /* 358 */ "alter_db_option", - /* 359 */ "retention", - /* 360 */ "full_table_name", - /* 361 */ "column_def_list", - /* 362 */ "tags_def_opt", - /* 363 */ "table_options", - /* 364 */ "multi_create_clause", - /* 365 */ "tags_def", - /* 366 */ "multi_drop_clause", - /* 367 */ "alter_table_clause", - /* 368 */ "alter_table_options", - /* 369 */ "column_name", - /* 370 */ "type_name", - /* 371 */ "signed_literal", - /* 372 */ "create_subtable_clause", - /* 373 */ "specific_cols_opt", - /* 374 */ "expression_list", - /* 375 */ "drop_table_clause", - /* 376 */ "col_name_list", - /* 377 */ "column_def", - /* 378 */ "duration_list", - /* 379 */ "rollup_func_list", - /* 380 */ "alter_table_option", - /* 381 */ "duration_literal", - /* 382 */ "rollup_func_name", - /* 383 */ "function_name", - /* 384 */ "col_name", - /* 385 */ "db_name_cond_opt", - /* 386 */ "like_pattern_opt", - /* 387 */ "table_name_cond", - /* 388 */ "from_db_opt", - /* 389 */ "tag_list_opt", - /* 390 */ "tag_item", - /* 391 */ "column_alias", - /* 392 */ "full_index_name", - /* 393 */ "index_options", - /* 394 */ "index_name", - /* 395 */ "func_list", - /* 396 */ "sliding_opt", - /* 397 */ "sma_stream_opt", - /* 398 */ "func", - /* 399 */ "sma_func_name", - /* 400 */ "query_or_subquery", - /* 401 */ "cgroup_name", - /* 402 */ "analyze_opt", - /* 403 */ "explain_options", - /* 404 */ "insert_query", - /* 405 */ "or_replace_opt", - /* 406 */ "agg_func_opt", - /* 407 */ "bufsize_opt", - /* 408 */ "language_opt", - /* 409 */ "stream_name", - /* 410 */ "stream_options", - /* 411 */ "col_list_opt", - /* 412 */ "tag_def_or_ref_opt", - /* 413 */ "subtable_opt", - /* 414 */ "expression", - /* 415 */ "dnode_list", - /* 416 */ "where_clause_opt", - /* 417 */ "signed", + /* 358 */ "signed", + /* 359 */ "alter_db_option", + /* 360 */ "retention", + /* 361 */ "full_table_name", + /* 362 */ "column_def_list", + /* 363 */ "tags_def_opt", + /* 364 */ "table_options", + /* 365 */ "multi_create_clause", + /* 366 */ "tags_def", + /* 367 */ "multi_drop_clause", + /* 368 */ "alter_table_clause", + /* 369 */ "alter_table_options", + /* 370 */ "column_name", + /* 371 */ "type_name", + /* 372 */ "signed_literal", + /* 373 */ "create_subtable_clause", + /* 374 */ "specific_cols_opt", + /* 375 */ "expression_list", + /* 376 */ "drop_table_clause", + /* 377 */ "col_name_list", + /* 378 */ "column_def", + /* 379 */ "duration_list", + /* 380 */ "rollup_func_list", + /* 381 */ "alter_table_option", + /* 382 */ "duration_literal", + /* 383 */ "rollup_func_name", + /* 384 */ "function_name", + /* 385 */ "col_name", + /* 386 */ "db_name_cond_opt", + /* 387 */ "like_pattern_opt", + /* 388 */ "table_name_cond", + /* 389 */ "from_db_opt", + /* 390 */ "tag_list_opt", + /* 391 */ "tag_item", + /* 392 */ "column_alias", + /* 393 */ "full_index_name", + /* 394 */ "index_options", + /* 395 */ "index_name", + /* 396 */ "func_list", + /* 397 */ "sliding_opt", + /* 398 */ "sma_stream_opt", + /* 399 */ "func", + /* 400 */ "sma_func_name", + /* 401 */ "query_or_subquery", + /* 402 */ "cgroup_name", + /* 403 */ "analyze_opt", + /* 404 */ "explain_options", + /* 405 */ "insert_query", + /* 406 */ "or_replace_opt", + /* 407 */ "agg_func_opt", + /* 408 */ "bufsize_opt", + /* 409 */ "language_opt", + /* 410 */ "stream_name", + /* 411 */ "stream_options", + /* 412 */ "col_list_opt", + /* 413 */ "tag_def_or_ref_opt", + /* 414 */ "subtable_opt", + /* 415 */ "expression", + /* 416 */ "dnode_list", + /* 417 */ "where_clause_opt", /* 418 */ "literal_func", /* 419 */ "literal_list", /* 420 */ "table_alias", @@ -2029,8 +2063,8 @@ static const char *const yyRuleName[] = { /* 106 */ "db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER", /* 107 */ "db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER", /* 108 */ "db_options ::= db_options STT_TRIGGER NK_INTEGER", - /* 109 */ "db_options ::= db_options TABLE_PREFIX NK_INTEGER", - /* 110 */ "db_options ::= db_options TABLE_SUFFIX NK_INTEGER", + /* 109 */ "db_options ::= db_options TABLE_PREFIX signed", + /* 110 */ "db_options ::= db_options TABLE_SUFFIX signed", /* 111 */ "alter_db_options ::= alter_db_option", /* 112 */ "alter_db_options ::= alter_db_options alter_db_option", /* 113 */ "alter_db_option ::= BUFFER NK_INTEGER", @@ -2637,36 +2671,36 @@ static void yy_destructor( case 351: /* alter_db_options */ case 353: /* start_opt */ case 354: /* end_opt */ - case 359: /* retention */ - case 360: /* full_table_name */ - case 363: /* table_options */ - case 367: /* alter_table_clause */ - case 368: /* alter_table_options */ - case 371: /* signed_literal */ - case 372: /* create_subtable_clause */ - case 375: /* drop_table_clause */ - case 377: /* column_def */ - case 381: /* duration_literal */ - case 382: /* rollup_func_name */ - case 384: /* col_name */ - case 385: /* db_name_cond_opt */ - case 386: /* like_pattern_opt */ - case 387: /* table_name_cond */ - case 388: /* from_db_opt */ - case 390: /* tag_item */ - case 392: /* full_index_name */ - case 393: /* index_options */ - case 396: /* sliding_opt */ - case 397: /* sma_stream_opt */ - case 398: /* func */ - case 400: /* query_or_subquery */ - case 403: /* explain_options */ - case 404: /* insert_query */ - case 410: /* stream_options */ - case 413: /* subtable_opt */ - case 414: /* expression */ - case 416: /* where_clause_opt */ - case 417: /* signed */ + case 358: /* signed */ + case 360: /* retention */ + case 361: /* full_table_name */ + case 364: /* table_options */ + case 368: /* alter_table_clause */ + case 369: /* alter_table_options */ + case 372: /* signed_literal */ + case 373: /* create_subtable_clause */ + case 376: /* drop_table_clause */ + case 378: /* column_def */ + case 382: /* duration_literal */ + case 383: /* rollup_func_name */ + case 385: /* col_name */ + case 386: /* db_name_cond_opt */ + case 387: /* like_pattern_opt */ + case 388: /* table_name_cond */ + case 389: /* from_db_opt */ + case 391: /* tag_item */ + case 393: /* full_index_name */ + case 394: /* index_options */ + case 397: /* sliding_opt */ + case 398: /* sma_stream_opt */ + case 399: /* func */ + case 401: /* query_or_subquery */ + case 404: /* explain_options */ + case 405: /* insert_query */ + case 411: /* stream_options */ + case 414: /* subtable_opt */ + case 415: /* expression */ + case 417: /* where_clause_opt */ case 418: /* literal_func */ case 421: /* expr_or_subquery */ case 422: /* pseudo_column */ @@ -2711,7 +2745,7 @@ static void yy_destructor( case 332: /* alter_account_options */ case 334: /* alter_account_option */ case 352: /* speed_opt */ - case 407: /* bufsize_opt */ + case 408: /* bufsize_opt */ { } @@ -2721,14 +2755,14 @@ static void yy_destructor( case 343: /* table_name */ case 344: /* topic_name */ case 346: /* dnode_endpoint */ - case 369: /* column_name */ - case 383: /* function_name */ - case 391: /* column_alias */ - case 394: /* index_name */ - case 399: /* sma_func_name */ - case 401: /* cgroup_name */ - case 408: /* language_opt */ - case 409: /* stream_name */ + case 370: /* column_name */ + case 384: /* function_name */ + case 392: /* column_alias */ + case 395: /* index_name */ + case 400: /* sma_func_name */ + case 402: /* cgroup_name */ + case 409: /* language_opt */ + case 410: /* stream_name */ case 420: /* table_alias */ case 426: /* star_func */ case 428: /* noarg_func */ @@ -2757,9 +2791,9 @@ static void yy_destructor( case 347: /* force_opt */ case 348: /* not_exists_opt */ case 350: /* exists_opt */ - case 402: /* analyze_opt */ - case 405: /* or_replace_opt */ - case 406: /* agg_func_opt */ + case 403: /* analyze_opt */ + case 406: /* or_replace_opt */ + case 407: /* agg_func_opt */ case 451: /* set_quantifier_opt */ { @@ -2768,21 +2802,21 @@ static void yy_destructor( case 355: /* integer_list */ case 356: /* variable_list */ case 357: /* retention_list */ - case 361: /* column_def_list */ - case 362: /* tags_def_opt */ - case 364: /* multi_create_clause */ - case 365: /* tags_def */ - case 366: /* multi_drop_clause */ - case 373: /* specific_cols_opt */ - case 374: /* expression_list */ - case 376: /* col_name_list */ - case 378: /* duration_list */ - case 379: /* rollup_func_list */ - case 389: /* tag_list_opt */ - case 395: /* func_list */ - case 411: /* col_list_opt */ - case 412: /* tag_def_or_ref_opt */ - case 415: /* dnode_list */ + case 362: /* column_def_list */ + case 363: /* tags_def_opt */ + case 365: /* multi_create_clause */ + case 366: /* tags_def */ + case 367: /* multi_drop_clause */ + case 374: /* specific_cols_opt */ + case 375: /* expression_list */ + case 377: /* col_name_list */ + case 379: /* duration_list */ + case 380: /* rollup_func_list */ + case 390: /* tag_list_opt */ + case 396: /* func_list */ + case 412: /* col_list_opt */ + case 413: /* tag_def_or_ref_opt */ + case 416: /* dnode_list */ case 419: /* literal_list */ case 427: /* star_func_para_list */ case 429: /* other_para_list */ @@ -2798,13 +2832,13 @@ static void yy_destructor( nodesDestroyList((yypminor->yy432)); } break; - case 358: /* alter_db_option */ - case 380: /* alter_table_option */ + case 359: /* alter_db_option */ + case 381: /* alter_table_option */ { } break; - case 370: /* type_name */ + case 371: /* type_name */ { } @@ -2958,15 +2992,18 @@ static YYACTIONTYPE yy_find_shift_action( do{ i = yy_shift_ofst[stateno]; assert( i>=0 ); - /* assert( i+YYNTOKEN<=(int)YY_NLOOKAHEAD ); */ + assert( i<=YY_ACTTAB_COUNT ); + assert( i+YYNTOKEN<=(int)YY_NLOOKAHEAD ); assert( iLookAhead!=YYNOCODE ); assert( iLookAhead < YYNTOKEN ); i += iLookAhead; - if( i>=YY_NLOOKAHEAD || yy_lookahead[i]!=iLookAhead ){ + assert( i<(int)YY_NLOOKAHEAD ); + if( yy_lookahead[i]!=iLookAhead ){ #ifdef YYFALLBACK YYCODETYPE iFallback; /* Fallback token */ - if( iLookAhead %s\n", @@ -2981,16 +3018,8 @@ static YYACTIONTYPE yy_find_shift_action( #ifdef YYWILDCARD { int j = i - iLookAhead + YYWILDCARD; - if( -#if YY_SHIFT_MIN+YYWILDCARD<0 - j>=0 && -#endif -#if YY_SHIFT_MAX+YYWILDCARD>=YY_ACTTAB_COUNT - j0 - ){ + assert( j<(int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0])) ); + if( yy_lookahead[j]==YYWILDCARD && iLookAhead>0 ){ #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n", @@ -3004,6 +3033,7 @@ static YYACTIONTYPE yy_find_shift_action( #endif /* YYWILDCARD */ return yy_default[stateno]; }else{ + assert( i>=0 && iyytos; #ifndef NDEBUG if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ - yysize = yyRuleInfo[yyruleno].nrhs; + yysize = yyRuleInfoNRhs[yyruleno]; if( yysize ){ - fprintf(yyTraceFILE, "%sReduce %d [%s], go to state %d.\n", + fprintf(yyTraceFILE, "%sReduce %d [%s]%s, pop back to state %d.\n", yyTracePrompt, - yyruleno, yyRuleName[yyruleno], yymsp[yysize].stateno); + yyruleno, yyRuleName[yyruleno], + yyrulenoyytos - yypParser->yystack)>yypParser->yyhwm ){ yypParser->yyhwm++; @@ -4183,12 +4800,12 @@ static YYACTIONTYPE yy_reduce( { yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_STT_TRIGGER, &yymsp[0].minor.yy0); } yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 109: /* db_options ::= db_options TABLE_PREFIX NK_INTEGER */ -{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_TABLE_PREFIX, &yymsp[0].minor.yy0); } + case 109: /* db_options ::= db_options TABLE_PREFIX signed */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_TABLE_PREFIX, yymsp[0].minor.yy448); } yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 110: /* db_options ::= db_options TABLE_SUFFIX NK_INTEGER */ -{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_TABLE_SUFFIX, &yymsp[0].minor.yy0); } + case 110: /* db_options ::= db_options TABLE_SUFFIX signed */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_TABLE_SUFFIX, yymsp[0].minor.yy448); } yymsp[-2].minor.yy448 = yylhsminor.yy448; break; case 111: /* alter_db_options ::= alter_db_option */ @@ -5444,9 +6061,9 @@ static YYACTIONTYPE yy_reduce( break; /********** End reduce actions ************************************************/ }; - assert( yyruleno Date: Fri, 21 Apr 2023 14:58:58 +0800 Subject: [PATCH 037/109] fix: add table_prefix/table_suffix cases --- source/libs/parser/src/parTranslater.c | 4 +- tests/parallel_test/cases.task | 1 + tests/script/tsim/db/table_prefix_suffix.sim | 182 +++++++++++++++++++ 3 files changed, 185 insertions(+), 2 deletions(-) create mode 100644 tests/script/tsim/db/table_prefix_suffix.sim diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index c0ee34d6aa..21ae3c74a2 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4181,13 +4181,13 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete static int32_t checkDbTbPrefixSuffixOptions(STranslateContext* pCxt, int32_t tbPrefix, int32_t tbSuffix) { if (tbPrefix < TSDB_MIN_HASH_PREFIX || tbPrefix > TSDB_MAX_HASH_PREFIX) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, - "Invalid option table_prefix: %d valid range: [%" PRId64 ", %" PRId64 "]", tbPrefix, + "Invalid option table_prefix: %d valid range: [%d, %d]", tbPrefix, TSDB_MIN_HASH_PREFIX, TSDB_MAX_HASH_PREFIX); } if (tbSuffix < TSDB_MIN_HASH_SUFFIX || tbSuffix > TSDB_MAX_HASH_SUFFIX) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, - "Invalid option table_suffix: %d valid range: [%" PRId64 ", %" PRId64 "]", tbSuffix, + "Invalid option table_suffix: %d valid range: [%d, %d]", tbSuffix, TSDB_MIN_HASH_SUFFIX, TSDB_MAX_HASH_SUFFIX); } diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index dda4ec3e84..6e662a9a15 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -746,6 +746,7 @@ ,,y,script,./test.sh -f tsim/db/show_create_table.sim ,,y,script,./test.sh -f tsim/db/tables.sim ,,y,script,./test.sh -f tsim/db/taosdlog.sim +,,y,script,./test.sh -f tsim/db/table_prefix_suffix.sim ,,y,script,./test.sh -f tsim/dnode/balance_replica1.sim ,,y,script,./test.sh -f tsim/dnode/balance_replica3.sim ,,y,script,./test.sh -f tsim/dnode/balance1.sim diff --git a/tests/script/tsim/db/table_prefix_suffix.sim b/tests/script/tsim/db/table_prefix_suffix.sim new file mode 100644 index 0000000000..1b483d7df7 --- /dev/null +++ b/tests/script/tsim/db/table_prefix_suffix.sim @@ -0,0 +1,182 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql drop database if exists db1; +sql create database db1 vgroups 5 TABLE_PREFIX 1 TABLE_SUFFIX 2; +sql use db1; +sql create table atb1aa (ts timestamp, f1 int); +sql create table btb1bb (ts timestamp, f1 int); +sql create table ctb1cc (ts timestamp, f1 int); +sql create table dtb1dd (ts timestamp, f1 int); +sql create table atb2aa (ts timestamp, f1 int); +sql create table btb2bb (ts timestamp, f1 int); +sql create table ctb2cc (ts timestamp, f1 int); +sql create table dtb2dd (ts timestamp, f1 int); +sql create table etb2ee (ts timestamp, f1 int); +sql show create database db1; +sql select count(*) a from information_schema.ins_tables where db_name='db1' group by vgroup_id having(count(*) > 0) order by a; +if $rows != 2 then + return -1 +endi +if $data00 != 4 then + return -1 +endi +if $data10 != 5 then + return -1 +endi +sql drop database if exists db1; + +sql drop database if exists db2; +sql create database db2 vgroups 5 TABLE_PREFIX -1 TABLE_SUFFIX -2; +sql use db2; +sql create table taaa11 (ts timestamp, f1 int); +sql create table tbbb11 (ts timestamp, f1 int); +sql create table tccc11 (ts timestamp, f1 int); +sql create table tddd11 (ts timestamp, f1 int); +sql create table taaa22 (ts timestamp, f1 int); +sql create table tbbb22 (ts timestamp, f1 int); +sql create table tccc22 (ts timestamp, f1 int); +sql create table tddd22 (ts timestamp, f1 int); +sql create table teee22 (ts timestamp, f1 int); +sql show create database db2; +sql select count(*) a from information_schema.ins_tables where db_name='db2' group by vgroup_id having(count(*) > 0) order by a; +if $rows != 2 then + return -1 +endi +if $data00 != 4 then + return -1 +endi +if $data10 != 5 then + return -1 +endi +sql drop database if exists db2; + +sql drop database if exists db3; +sql create database db3 vgroups 5 TABLE_PREFIX -1; +sql use db3; +sql create table taaa11 (ts timestamp, f1 int); +sql create table tbbb11 (ts timestamp, f1 int); +sql create table tccc11 (ts timestamp, f1 int); +sql create table tddd11 (ts timestamp, f1 int); +sql create table zaaa22 (ts timestamp, f1 int); +sql create table zbbb22 (ts timestamp, f1 int); +sql create table zccc22 (ts timestamp, f1 int); +sql create table zddd22 (ts timestamp, f1 int); +sql create table zeee22 (ts timestamp, f1 int); +sql show create database db3; +sql select count(*) a from information_schema.ins_tables where db_name='db3' group by vgroup_id having(count(*) > 0) order by a; +if $rows != 2 then + return -1 +endi +if $data00 != 4 then + return -1 +endi +if $data10 != 5 then + return -1 +endi +sql drop database if exists db3; + +sql drop database if exists db4; +sql create database db4 vgroups 5 TABLE_SUFFIX -2; +sql use db4; +sql create table taaa11 (ts timestamp, f1 int); +sql create table tbbb11 (ts timestamp, f1 int); +sql create table tccc11 (ts timestamp, f1 int); +sql create table tddd11 (ts timestamp, f1 int); +sql create table zaaa22 (ts timestamp, f1 int); +sql create table zbbb22 (ts timestamp, f1 int); +sql create table zccc22 (ts timestamp, f1 int); +sql create table zddd22 (ts timestamp, f1 int); +sql create table zeee22 (ts timestamp, f1 int); +sql show create database db4; +sql select count(*) a from information_schema.ins_tables where db_name='db4' group by vgroup_id having(count(*) > 0) order by a; +if $rows != 2 then + return -1 +endi +if $data00 != 4 then + return -1 +endi +if $data10 != 5 then + return -1 +endi +sql drop database if exists db4; + +sql drop database if exists db5; +sql create database db5 vgroups 5 TABLE_PREFIX 1; +sql use db5; +sql create table taaa11 (ts timestamp, f1 int); +sql create table baaa11 (ts timestamp, f1 int); +sql create table caaa11 (ts timestamp, f1 int); +sql create table daaa11 (ts timestamp, f1 int); +sql create table faaa11 (ts timestamp, f1 int); +sql create table gbbb11 (ts timestamp, f1 int); +sql create table hbbb11 (ts timestamp, f1 int); +sql create table ibbb11 (ts timestamp, f1 int); +sql create table jbbb11 (ts timestamp, f1 int); +sql show create database db5; +sql select count(*) a from information_schema.ins_tables where db_name='db5' group by vgroup_id having(count(*) > 0) order by a; +if $rows != 2 then + return -1 +endi +if $data00 != 4 then + return -1 +endi +if $data10 != 5 then + return -1 +endi +sql drop database if exists db5; + +sql drop database if exists db6; +sql create database db6 vgroups 5 TABLE_SUFFIX 2; +sql use db6; +sql create table taaa11 (ts timestamp, f1 int); +sql create table taaa12 (ts timestamp, f1 int); +sql create table taaa13 (ts timestamp, f1 int); +sql create table taaa14 (ts timestamp, f1 int); +sql create table tbbb23 (ts timestamp, f1 int); +sql create table tbbb24 (ts timestamp, f1 int); +sql create table tbbb31 (ts timestamp, f1 int); +sql create table tbbb32 (ts timestamp, f1 int); +sql create table tbbb33 (ts timestamp, f1 int); +sql show create database db6; +sql select count(*) a from information_schema.ins_tables where db_name='db6' group by vgroup_id having(count(*) > 0) order by a; +if $rows != 2 then + return -1 +endi +if $data00 != 4 then + return -1 +endi +if $data10 != 5 then + return -1 +endi +sql drop database if exists db6; + +sql drop database if exists db7; +sql create database db7 vgroups 5 TABLE_PREFIX -100 TABLE_SUFFIX -92; +sql use db7; +sql create table taaa11 (ts timestamp, f1 int); +sql create table taaa12 (ts timestamp, f1 int); +sql create table taaa13 (ts timestamp, f1 int); +sql create table tbbb21 (ts timestamp, f1 int); +sql create table tbbb22 (ts timestamp, f1 int); +sql create table tbbb23 (ts timestamp, f1 int); +sql create table tbbb24 (ts timestamp, f1 int); +sql create table tccc31 (ts timestamp, f1 int); +sql create table tccc32 (ts timestamp, f1 int); +sql create table tccc33 (ts timestamp, f1 int); +sql create table tddd24 (ts timestamp, f1 int); +sql create table tddd31 (ts timestamp, f1 int); +sql create table tddd32 (ts timestamp, f1 int); +sql create table tddd33 (ts timestamp, f1 int); +sql show create database db7; +sql select count(*) a from information_schema.ins_tables where db_name='db7' group by vgroup_id having(count(*) > 0) order by a; +sql drop database if exists db7; + +sql_error create database db8 vgroups 5 TABLE_PREFIX -1 TABLE_SUFFIX 2; +sql_error create database db8 vgroups 5 TABLE_PREFIX 191 TABLE_SUFFIX 192; +sql_error create database db8 vgroups 5 TABLE_PREFIX -192 TABLE_SUFFIX -191; +sql_error create database db8 vgroups 5 TABLE_PREFIX 100 TABLE_SUFFIX 92; + +system sh/exec.sh -n dnode1 -s stop -x SIGINT From 5531775f8efc1aa70a3b2f86a661c3fbae02709a Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 21 Apr 2023 14:59:35 +0800 Subject: [PATCH 038/109] fix: when db_name!=xxx is used in systable scan index --- source/libs/executor/src/sysscanoperator.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 1abe678ac6..b0b27e0e67 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -1482,11 +1482,7 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) { pInfo->pIdx->init = 1; SSDataBlock* blk = sysTableBuildUserTablesByUids(pOperator); return blk; - } else if (flt == -2) { - qDebug("%s failed to get sys table info by idx, empty result", GET_TASKID(pTaskInfo)); - return NULL; - } else if (flt == -1) { - // not idx + } else if ((flt == -2) || (flt == -1)) { qDebug("%s failed to get sys table info by idx, scan sys table one by one", GET_TASKID(pTaskInfo)); } } else if (pCondition != NULL && (pInfo->pIdx != NULL && pInfo->pIdx->init == 1)) { From c971fe16062f0de5f1f273170f0680f19d6efec3 Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 21 Apr 2023 15:01:35 +0800 Subject: [PATCH 039/109] fix: when db_name != xxx can not be optimized by uid index --- source/libs/executor/src/sysscanoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index b0b27e0e67..c78e6002cd 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -1482,7 +1482,7 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) { pInfo->pIdx->init = 1; SSDataBlock* blk = sysTableBuildUserTablesByUids(pOperator); return blk; - } else if ((flt == -2) || (flt == -1)) { + } else if ((flt == -1) || (flt == -2)) { qDebug("%s failed to get sys table info by idx, scan sys table one by one", GET_TASKID(pTaskInfo)); } } else if (pCondition != NULL && (pInfo->pIdx != NULL && pInfo->pIdx->init == 1)) { From d0b370e736de6af184bb6cc79a589a8daaa16e4a Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 21 Apr 2023 16:28:25 +0800 Subject: [PATCH 040/109] fix: add test case --- tests/script/tsim/query/sys_tbname.sim | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/script/tsim/query/sys_tbname.sim b/tests/script/tsim/query/sys_tbname.sim index c676f2b1e0..849aeb2ac5 100644 --- a/tests/script/tsim/query/sys_tbname.sim +++ b/tests/script/tsim/query/sys_tbname.sim @@ -109,5 +109,26 @@ if $rows != 5000 then return -1 endi +sql create database d1; +sql create stable d1.st1 (ts timestamp, f int) tags(t int); +sql create stable d1.st2 (ts timestamp, f int) tags(t int); +sql create table d1.ct1 using d1.st1 tags(1); +sql create table d1.ct2 using d1.st2 tags(2); +sql create database d2; +sql create stable d2.st1(ts timestamp, f int) tags(t int); +sql create stable d2.st2(ts timestamp, f int) tags(t int); +sql create table d2.ct1 using d2.st1 tags(1); +sql create table d2.ct2 using d2.st2 tags(2); + +sql create database d3; +sql create stable d3.st1(ts timestamp, f int) tags(t int); +sql create stable d3.st2(ts timestamp, f int) tags(t int); +sql create table d3.ct1 using d3.st1 tags(1); +sql create table d3.ct2 using d3.st2 tags(2); +sql select count(*), stable_name, db_name from information_schema.ins_tables where db_name != 'd2' group by stable_name,db_name +print $rows +if $rows != 9 then + return -1 +endi #system sh/exec.sh -n dnode1 -s stop -x SIGINT From 176f7740c34a0d8372cda791450b986e28052d4d Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 21 Apr 2023 16:41:39 +0800 Subject: [PATCH 041/109] fix: memory leak issue --- source/libs/parser/src/parAstCreater.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 0ba57af29c..5a47ed731d 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -1032,6 +1032,7 @@ static SNode* setDatabaseOptionImpl(SAstCreateContext* pCxt, SNode* pOptions, ED snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid table_prefix data type"); pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; } + nodesDestroyNode((SNode*)pNode); break; } case DB_OPTION_TABLE_SUFFIX:{ @@ -1042,6 +1043,7 @@ static SNode* setDatabaseOptionImpl(SAstCreateContext* pCxt, SNode* pOptions, ED snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid table_suffix data type"); pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; } + nodesDestroyNode((SNode*)pNode); break; } default: From 7ea860ad1111fdc1db9849444f1ab4a7d24e0337 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 21 Apr 2023 17:14:41 +0800 Subject: [PATCH 042/109] fix:[TD-23714] topic support capital --- source/client/src/clientTmq.c | 3 --- tests/system-test/7-tmq/stbTagFilter-1ctb.py | 4 ++-- tests/system-test/7-tmq/stbTagFilter-multiCtb.py | 4 ++-- tests/system-test/7-tmq/tmqDropStb.py | 2 +- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index ceca06e309..fd6dc918a9 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -373,9 +373,6 @@ int32_t tmq_list_append(tmq_list_t* list, const char* src) { SArray* container = &list->container; if (src == NULL || src[0] == 0) return -1; char* topic = taosStrdup(src); - if (topic[0] != '`') { - strtolower(topic, src); - } if (taosArrayPush(container, &topic) == NULL) return -1; return 0; } diff --git a/tests/system-test/7-tmq/stbTagFilter-1ctb.py b/tests/system-test/7-tmq/stbTagFilter-1ctb.py index 7ee5fce5a8..d226b18d17 100644 --- a/tests/system-test/7-tmq/stbTagFilter-1ctb.py +++ b/tests/system-test/7-tmq/stbTagFilter-1ctb.py @@ -111,7 +111,7 @@ class TDTestCase: topicFromStb1 = 'topic_UpperCase_stb1' # queryString = "select ts, c1, c2 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) queryString = "select ts, c1, c2, t4 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) - sqlString = "create topic %s as %s" %(topicFromStb1, queryString) + sqlString = "create topic `%s` as %s" %(topicFromStb1, queryString) tdLog.info("create topic sql: %s"%sqlString) tdSql.execute(sqlString) @@ -196,7 +196,7 @@ class TDTestCase: topicFromStb1 = 'topic_UpperCase_stb1' queryString = "select ts, c1, c2 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) # queryString = "select ts, c1, c2, t4 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) - sqlString = "create topic %s as %s" %(topicFromStb1, queryString) + sqlString = "create topic `%s` as %s" %(topicFromStb1, queryString) tdLog.info("create topic sql: %s"%sqlString) tdSql.execute(sqlString) diff --git a/tests/system-test/7-tmq/stbTagFilter-multiCtb.py b/tests/system-test/7-tmq/stbTagFilter-multiCtb.py index 71b7fdef5d..5ae821ad72 100644 --- a/tests/system-test/7-tmq/stbTagFilter-multiCtb.py +++ b/tests/system-test/7-tmq/stbTagFilter-multiCtb.py @@ -111,7 +111,7 @@ class TDTestCase: topicFromStb1 = 'topic_UpperCase_stb1' queryString = "select ts, c1, c2 from %s.%s where t4 == 'beijing' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) # queryString = "select ts, c1, c2, t4 from %s.%s where t4 == 'beijing' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) - sqlString = "create topic %s as %s" %(topicFromStb1, queryString) + sqlString = "create topic `%s` as %s" %(topicFromStb1, queryString) tdLog.info("create topic sql: %s"%sqlString) tdSql.execute(sqlString) @@ -196,7 +196,7 @@ class TDTestCase: topicFromStb1 = 'topic_UpperCase_stb1' # queryString = "select ts, c1, c2 from %s.%s where t4 == 'beijing' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) queryString = "select ts, c1, c2, t4 from %s.%s where t4 == 'beijing' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) - sqlString = "create topic %s as %s" %(topicFromStb1, queryString) + sqlString = "create topic `%s` as %s" %(topicFromStb1, queryString) tdLog.info("create topic sql: %s"%sqlString) tdSql.execute(sqlString) diff --git a/tests/system-test/7-tmq/tmqDropStb.py b/tests/system-test/7-tmq/tmqDropStb.py index a94747e574..8680bdf09e 100644 --- a/tests/system-test/7-tmq/tmqDropStb.py +++ b/tests/system-test/7-tmq/tmqDropStb.py @@ -82,7 +82,7 @@ class TDTestCase: tdLog.info("create topics from db") topicName1 = 'UpperCasetopic_%s'%(self.paraDict['dbName']) - tdSql.execute("create topic %s as database %s" %(topicName1, self.paraDict['dbName'])) + tdSql.execute("create topic `%s` as database %s" %(topicName1, self.paraDict['dbName'])) topicList = topicName1 + ',' +topicName1 keyList = '%s,%s,%s,%s'%(self.groupId,self.autoCommit,self.autoCommitInterval,self.autoOffset) From d288e44f921fb5b641391ad41a8f01c898a45234 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 21 Apr 2023 17:18:40 +0800 Subject: [PATCH 043/109] fix:ci error --- source/client/src/clientTmq.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 16a4f55840..6642e716e6 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -373,6 +373,9 @@ int32_t tmq_list_append(tmq_list_t* list, const char* src) { SArray* container = &list->container; if (src == NULL || src[0] == 0) return -1; char* topic = taosStrdup(src); + if (topic[0] != '`') { + strtolower(topic, src); + } if (taosArrayPush(container, &topic) == NULL) return -1; return 0; } From 0835812f04e403c017abeed7442c4be67fca5514 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 21 Apr 2023 18:16:17 +0800 Subject: [PATCH 044/109] fix:[TD-23714] topic support capital --- tests/system-test/7-tmq/stbTagFilter-1ctb.py | 4 ++-- tests/system-test/7-tmq/stbTagFilter-multiCtb.py | 4 ++-- tests/system-test/7-tmq/tmqDropStb.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/system-test/7-tmq/stbTagFilter-1ctb.py b/tests/system-test/7-tmq/stbTagFilter-1ctb.py index d226b18d17..1867dc54cb 100644 --- a/tests/system-test/7-tmq/stbTagFilter-1ctb.py +++ b/tests/system-test/7-tmq/stbTagFilter-1ctb.py @@ -148,7 +148,7 @@ class TDTestCase: tmqCom.checkFileContent(consumerId, queryString) - tdSql.query("drop topic %s"%topicFromStb1) + tdSql.query("drop topic `%s`"%topicFromStb1) tdLog.printNoPrefix("======== test case 1 end ...... ") def tmqCase2(self): @@ -242,7 +242,7 @@ class TDTestCase: # tmqCom.checkFileContent(consumerId, queryString) - tdSql.query("drop topic %s"%topicFromStb1) + tdSql.query("drop topic `%s`"%topicFromStb1) tdLog.printNoPrefix("======== test case 2 end ...... ") diff --git a/tests/system-test/7-tmq/stbTagFilter-multiCtb.py b/tests/system-test/7-tmq/stbTagFilter-multiCtb.py index 5ae821ad72..67cc60d196 100644 --- a/tests/system-test/7-tmq/stbTagFilter-multiCtb.py +++ b/tests/system-test/7-tmq/stbTagFilter-multiCtb.py @@ -148,7 +148,7 @@ class TDTestCase: # tmqCom.checkFileContent(consumerId, queryString) - tdSql.query("drop topic %s"%topicFromStb1) + tdSql.query("drop topic `%s`"%topicFromStb1) tdLog.printNoPrefix("======== test case 1 end ...... ") def tmqCase2(self): @@ -244,7 +244,7 @@ class TDTestCase: # tmqCom.checkFileContent(consumerId, queryString) - tdSql.query("drop topic %s"%topicFromStb1) + tdSql.query("drop topic `%s`"%topicFromStb1) tdLog.printNoPrefix("======== test case 2 end ...... ") diff --git a/tests/system-test/7-tmq/tmqDropStb.py b/tests/system-test/7-tmq/tmqDropStb.py index 8680bdf09e..0b252a7334 100644 --- a/tests/system-test/7-tmq/tmqDropStb.py +++ b/tests/system-test/7-tmq/tmqDropStb.py @@ -113,7 +113,7 @@ class TDTestCase: tdLog.exit("tmq consume rows error!") time.sleep(10) - tdSql.query("drop topic %s"%topicName1) + tdSql.query("drop topic `%s`"%topicName1) tdLog.printNoPrefix("======== test case 1 end ...... ") From 3c5351629939f2b49592686964e35387e4e82059 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 21 Apr 2023 19:23:46 +0800 Subject: [PATCH 045/109] fix: use tdbTbUpsert for metaUpdate functions --- source/dnode/vnode/src/meta/metaTable.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 8a311b9b80..2b5bc36acb 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -1029,7 +1029,7 @@ int metaUpdateCtimeIdx(SMeta *pMeta, const SMetaEntry *pME) { metaTrace("vgId:%d, start to save version:%" PRId64 " uid:%" PRId64 " ctime:%" PRId64, TD_VID(pMeta->pVnode), pME->version, pME->uid, ctimeKey.ctime); - return tdbTbInsert(pMeta->pCtimeIdx, &ctimeKey, sizeof(ctimeKey), NULL, 0, pMeta->txn); + return tdbTbUpsert(pMeta->pCtimeIdx, &ctimeKey, sizeof(ctimeKey), NULL, 0, pMeta->txn); } int metaDeleteCtimeIdx(SMeta *pMeta, const SMetaEntry *pME) { @@ -1044,7 +1044,7 @@ int metaUpdateNcolIdx(SMeta *pMeta, const SMetaEntry *pME) { if (metaBuildNColIdxKey(&ncolKey, pME) < 0) { return 0; } - return tdbTbInsert(pMeta->pNcolIdx, &ncolKey, sizeof(ncolKey), NULL, 0, pMeta->txn); + return tdbTbUpsert(pMeta->pNcolIdx, &ncolKey, sizeof(ncolKey), NULL, 0, pMeta->txn); } int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME) { @@ -1878,24 +1878,24 @@ static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) { } static int metaUpdateSuidIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbTbInsert(pMeta->pSuidIdx, &pME->uid, sizeof(tb_uid_t), NULL, 0, pMeta->txn); + return tdbTbUpsert(pMeta->pSuidIdx, &pME->uid, sizeof(tb_uid_t), NULL, 0, pMeta->txn); } static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbTbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), pMeta->txn); + return tdbTbUpsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), pMeta->txn); } static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) { STtlIdxKey ttlKey = {0}; metaBuildTtlIdxKey(&ttlKey, pME); if (ttlKey.dtime == 0) return 0; - return tdbTbInsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, pMeta->txn); + return tdbTbUpsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, pMeta->txn); } static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) { SCtbIdxKey ctbIdxKey = {.suid = pME->ctbEntry.suid, .uid = pME->uid}; - return tdbTbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), pME->ctbEntry.pTags, + return tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), pME->ctbEntry.pTags, ((STag *)(pME->ctbEntry.pTags))->len, pMeta->txn); } @@ -2114,12 +2114,14 @@ int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME) { } metaULock(pMeta); + metaDebug("vgId:%d, handle meta entry, ver:%" PRId64 ", uid:%" PRId64 ", name:%s", TD_VID(pMeta->pVnode), + pME->version, pME->uid, pME->name); return 0; _err: metaULock(pMeta); - metaError("vgId:%d, failed to handle meta entry since %s at line:%d, ver:%" PRId64 ", uid:%" PRId64, - TD_VID(pMeta->pVnode), terrstr(), line, pME->version, pME->uid); + metaError("vgId:%d, failed to handle meta entry since %s at line:%d, ver:%" PRId64 ", uid:%" PRId64 ", name:%s", + TD_VID(pMeta->pVnode), terrstr(), line, pME->version, pME->uid, pME->name); return -1; } From 1e9d999988154db616bbb104200dc8ec423b985e Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 22 Apr 2023 16:51:39 +0800 Subject: [PATCH 046/109] fix: taosdump escape dbname (#21014) * fix: taosdump escape dbname * fix: json file for escape char * fix: update taostools ffc2e6f --- cmake/taostools_CMakeLists.txt.in | 2 +- .../taosbenchmark/json/rest_auto_create_table.json | 3 +-- .../5-taos-tools/taosbenchmark/json/sml_auto_create_table.json | 3 +-- .../taosbenchmark/json/stmt_auto_create_table.json | 3 +-- .../taosbenchmark/json/taosc_auto_create_table.json | 3 +-- 5 files changed, 5 insertions(+), 9 deletions(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index d8bf3a09b4..4838e97dd7 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 0681d8b + GIT_TAG ffc2e6f SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/rest_auto_create_table.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/rest_auto_create_table.json index 9b99521f52..078688bb7e 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/json/rest_auto_create_table.json +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/rest_auto_create_table.json @@ -11,6 +11,7 @@ "confirm_parameter_prompt": "no", "prepared_rand": 100, "chinese": "no", + "escape_character": "yes", "insert_interval": 0, "num_of_records_per_req": 10, "databases": [{ @@ -29,7 +30,6 @@ "child_table_exists":"no", "childtable_count": 8, "childtable_prefix": "stb_", - "escape_character": "yes", "auto_create_table": "yes", "batch_create_tbl_num": 10, "data_source": "rand", @@ -54,7 +54,6 @@ "child_table_exists":"no", "childtable_count": 8, "childtable_prefix": "stb3-2_", - "escape_character": "yes", "auto_create_table": "yes", "batch_create_tbl_num": 10, "data_source": "rand", diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/sml_auto_create_table.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/sml_auto_create_table.json index 7b87919a6d..e808c9d3f3 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/json/sml_auto_create_table.json +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/sml_auto_create_table.json @@ -11,6 +11,7 @@ "confirm_parameter_prompt": "no", "prepared_rand": 100, "chinese": "no", + "escape_character": "yes", "insert_interval": 0, "num_of_records_per_req": 10, "databases": [{ @@ -29,7 +30,6 @@ "child_table_exists":"no", "childtable_count": 8, "childtable_prefix": "stb_", - "escape_character": "yes", "auto_create_table": "yes", "batch_create_tbl_num": 10, "data_source": "rand", @@ -54,7 +54,6 @@ "child_table_exists":"no", "childtable_count": 8, "childtable_prefix": "stb4-2_", - "escape_character": "yes", "auto_create_table": "yes", "batch_create_tbl_num": 10, "data_source": "rand", diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/stmt_auto_create_table.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/stmt_auto_create_table.json index baf0384e46..0054d985ee 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/json/stmt_auto_create_table.json +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/stmt_auto_create_table.json @@ -11,6 +11,7 @@ "confirm_parameter_prompt": "no", "prepared_rand": 100, "chinese": "no", + "escape_character": "yes", "insert_interval": 0, "num_of_records_per_req": 10, "databases": [{ @@ -29,7 +30,6 @@ "child_table_exists":"no", "childtable_count": 8, "childtable_prefix": "stb_", - "escape_character": "yes", "auto_create_table": "yes", "batch_create_tbl_num": 10, "data_source": "rand", @@ -54,7 +54,6 @@ "child_table_exists":"no", "childtable_count": 8, "childtable_prefix": "stb2-2_", - "escape_character": "yes", "auto_create_table": "yes", "batch_create_tbl_num": 10, "data_source": "rand", diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/taosc_auto_create_table.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/taosc_auto_create_table.json index f683cc016b..bed3598bfe 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/json/taosc_auto_create_table.json +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/taosc_auto_create_table.json @@ -11,6 +11,7 @@ "confirm_parameter_prompt": "no", "prepared_rand": 100, "chinese": "no", + "escape_character": "yes", "insert_interval": 0, "num_of_records_per_req": 10, "databases": [{ @@ -29,7 +30,6 @@ "child_table_exists":"no", "childtable_count": 8, "childtable_prefix": "stb_", - "escape_character": "yes", "auto_create_table": "yes", "batch_create_tbl_num": 10, "data_source": "rand", @@ -55,7 +55,6 @@ "child_table_exists":"no", "childtable_count": 8, "childtable_prefix": "stb1-2_", - "escape_character": "yes", "auto_create_table": "yes", "batch_create_tbl_num": 10, "data_source": "rand", From 5c9450856ec3ac70ed0d9a8789445e027c038c73 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 23 Apr 2023 00:53:55 +0000 Subject: [PATCH 047/109] fix filter err --- source/libs/index/src/indexFilter.c | 72 ++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index ec5fc5ad2a..bc86de5aeb 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -221,6 +221,71 @@ static FORCE_INLINE int32_t sifInitJsonParam(SNode *node, SIFParam *param, SIFCt param->status = SFLT_COARSE_INDEX; return 0; } +static int32_t sifNeedConvertCond(SNode *l, SNode *r) { + if (nodeType(l) != QUERY_NODE_COLUMN || nodeType(r) != QUERY_NODE_VALUE) { + return 0; + } + SColumnNode *c = (SColumnNode *)l; + SValueNode *v = (SValueNode *)r; + int32_t ctype = c->node.resType.type; + int32_t vtype = v->node.resType.type; + if (!IS_VAR_DATA_TYPE(ctype) && IS_VAR_DATA_TYPE(vtype)) { + return 1; + } + return 0; +} +static int32_t sifInitParamValByCol(SNode *r, SNode *l, SIFParam *param, SIFCtx *ctx) { + param->status = SFLT_COARSE_INDEX; + SColumnNode *cn = (SColumnNode *)r; + SValueNode *vn = (SValueNode *)l; + if (vn->typeData == TSDB_DATA_TYPE_NULL && (vn->literal == NULL || strlen(vn->literal) == 0)) { + param->status = SFLT_NOT_INDEX; + return 0; + } + SDataType *pType = &cn->node.resType; + int32_t type = pType->type; + + SDataType *pVType = &vn->node.resType; + int32_t vtype = pVType->type; + char *pData = nodesGetValueFromNode(vn); + int32_t valLen = 0; + char **value = ¶m->condValue; + + if (IS_VAR_DATA_TYPE(type)) { + int32_t dataLen = varDataTLen(pData); + if (type == TSDB_DATA_TYPE_JSON) { + if (*pData == TSDB_DATA_TYPE_NULL) { + dataLen = 0; + } else if (*pData == TSDB_DATA_TYPE_NCHAR) { + dataLen = varDataTLen(pData); + } else if (*pData == TSDB_DATA_TYPE_DOUBLE) { + dataLen = LONG_BYTES; + } else if (*pData == TSDB_DATA_TYPE_BOOL) { + dataLen = CHAR_BYTES; + } + dataLen += CHAR_BYTES; + } + valLen = dataLen; + } else { + valLen = pType->bytes; + } + char *tv = taosMemoryCalloc(1, valLen + 1); + if (tv == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + memcpy(tv, pData, valLen); + *value = tv; + + param->colId = -1; + param->colValType = (uint8_t)(vn->node.resType.type); + if (vn->literal != NULL && strlen(vn->literal) <= sizeof(param->colName)) { + memcpy(param->colName, vn->literal, strlen(vn->literal)); + } else { + param->status = SFLT_NOT_INDEX; + } + return 0; +} static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { param->status = SFLT_COARSE_INDEX; switch (nodeType(node)) { @@ -317,8 +382,13 @@ static int32_t sifInitOperParams(SIFParam **params, SOperatorNode *node, SIFCtx return TSDB_CODE_SUCCESS; } else { SIF_ERR_JRET(sifInitParam(node->pLeft, ¶mList[0], ctx)); + if (nParam > 1) { - SIF_ERR_JRET(sifInitParam(node->pRight, ¶mList[1], ctx)); + if (sifNeedConvertCond(node->pLeft, node->pRight)) { + SIF_ERR_JRET(sifInitParamValByCol(node->pLeft, node->pRight, ¶mList[1], ctx)); + } else { + SIF_ERR_JRET(sifInitParam(node->pRight, ¶mList[1], ctx)); + } // if (paramList[0].colValType == TSDB_DATA_TYPE_JSON && // ((SOperatorNode *)(node))->opType == OP_TYPE_JSON_CONTAINS) { // return TSDB_CODE_OUT_OF_MEMORY; From 3b89aa2085a15c090d10766474a02e1daeb88a4b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 23 Apr 2023 02:10:45 +0000 Subject: [PATCH 048/109] fix filter err --- source/libs/index/src/indexFilter.c | 105 +++++++++++++++------------- 1 file changed, 55 insertions(+), 50 deletions(-) diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index bc86de5aeb..23751cf26b 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -477,56 +477,61 @@ static FORCE_INLINE FilterFunc sifGetFilterFunc(EIndexQueryType type, bool *reve static void sifSetFltParam(SIFParam *left, SIFParam *right, SDataTypeBuf *typedata, SMetaFltParam *param) { int8_t ltype = left->colValType, rtype = right->colValType; - if (ltype == TSDB_DATA_TYPE_FLOAT) { - float f = 0; - SIF_DATA_CONVERT(rtype, right->condValue, f); - typedata->f = f; - param->val = &typedata->f; - } else if (ltype == TSDB_DATA_TYPE_DOUBLE) { - double d = 0; - SIF_DATA_CONVERT(rtype, right->condValue, d); - typedata->d = d; - param->val = &typedata->d; - } else if (ltype == TSDB_DATA_TYPE_BIGINT) { - int64_t i64 = 0; - SIF_DATA_CONVERT(rtype, right->condValue, i64); - typedata->i64 = i64; - param->val = &typedata->i64; - } else if (ltype == TSDB_DATA_TYPE_INT) { - int32_t i32 = 0; - SIF_DATA_CONVERT(rtype, right->condValue, i32); - typedata->i32 = i32; - param->val = &typedata->i32; - } else if (ltype == TSDB_DATA_TYPE_SMALLINT) { - int16_t i16 = 0; - SIF_DATA_CONVERT(rtype, right->condValue, i16); - typedata->i16 = i16; - param->val = &typedata->i16; - } else if (ltype == TSDB_DATA_TYPE_TINYINT) { - int8_t i8 = 0; - SIF_DATA_CONVERT(rtype, right->condValue, i8) - typedata->i8 = i8; - param->val = &typedata->i8; - } else if (ltype == TSDB_DATA_TYPE_UBIGINT) { - uint64_t u64 = 0; - SIF_DATA_CONVERT(rtype, right->condValue, u64); - typedata->u64 = u64; - param->val = &typedata->u64; - } else if (ltype == TSDB_DATA_TYPE_UINT) { - uint32_t u32 = 0; - SIF_DATA_CONVERT(rtype, right->condValue, u32); - typedata->u32 = u32; - param->val = &typedata->u32; - } else if (ltype == TSDB_DATA_TYPE_USMALLINT) { - uint16_t u16 = 0; - SIF_DATA_CONVERT(rtype, right->condValue, u16); - typedata->u16 = u16; - param->val = &typedata->u16; - } else if (ltype == TSDB_DATA_TYPE_UTINYINT) { - uint8_t u8 = 0; - SIF_DATA_CONVERT(rtype, right->condValue, u8); - typedata->u8 = u8; - param->val = &typedata->u8; + if (!IS_VAR_DATA_TYPE(ltype) && IS_VAR_DATA_TYPE(rtype)) { + } else if (IS_VAR_DATA_TYPE(ltype) && !IS_VAR_DATA_TYPE(rtype)) { + return; + } else if (!IS_VAR_DATA_TYPE(ltype) && !IS_VAR_DATA_TYPE(rtype)) { + if (ltype == TSDB_DATA_TYPE_FLOAT) { + float f = 0; + SIF_DATA_CONVERT(rtype, right->condValue, f); + typedata->f = f; + param->val = &typedata->f; + } else if (ltype == TSDB_DATA_TYPE_DOUBLE) { + double d = 0; + SIF_DATA_CONVERT(rtype, right->condValue, d); + typedata->d = d; + param->val = &typedata->d; + } else if (ltype == TSDB_DATA_TYPE_BIGINT) { + int64_t i64 = 0; + SIF_DATA_CONVERT(rtype, right->condValue, i64); + typedata->i64 = i64; + param->val = &typedata->i64; + } else if (ltype == TSDB_DATA_TYPE_INT) { + int32_t i32 = 0; + SIF_DATA_CONVERT(rtype, right->condValue, i32); + typedata->i32 = i32; + param->val = &typedata->i32; + } else if (ltype == TSDB_DATA_TYPE_SMALLINT) { + int16_t i16 = 0; + SIF_DATA_CONVERT(rtype, right->condValue, i16); + typedata->i16 = i16; + param->val = &typedata->i16; + } else if (ltype == TSDB_DATA_TYPE_TINYINT) { + int8_t i8 = 0; + SIF_DATA_CONVERT(rtype, right->condValue, i8) + typedata->i8 = i8; + param->val = &typedata->i8; + } else if (ltype == TSDB_DATA_TYPE_UBIGINT) { + uint64_t u64 = 0; + SIF_DATA_CONVERT(rtype, right->condValue, u64); + typedata->u64 = u64; + param->val = &typedata->u64; + } else if (ltype == TSDB_DATA_TYPE_UINT) { + uint32_t u32 = 0; + SIF_DATA_CONVERT(rtype, right->condValue, u32); + typedata->u32 = u32; + param->val = &typedata->u32; + } else if (ltype == TSDB_DATA_TYPE_USMALLINT) { + uint16_t u16 = 0; + SIF_DATA_CONVERT(rtype, right->condValue, u16); + typedata->u16 = u16; + param->val = &typedata->u16; + } else if (ltype == TSDB_DATA_TYPE_UTINYINT) { + uint8_t u8 = 0; + SIF_DATA_CONVERT(rtype, right->condValue, u8); + typedata->u8 = u8; + param->val = &typedata->u8; + } } } static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) { From c86f0a331ae2c60783b177ffb484e0469d8e419b Mon Sep 17 00:00:00 2001 From: cadem Date: Sun, 23 Apr 2023 10:23:54 +0800 Subject: [PATCH 049/109] fix:add log for locale setting --- source/os/src/osLocale.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/os/src/osLocale.c b/source/os/src/osLocale.c index 7008c38576..129faaacc8 100644 --- a/source/os/src/osLocale.c +++ b/source/os/src/osLocale.c @@ -171,7 +171,7 @@ void taosGetSystemLocale(char *outLocale, char *outCharset) { strcpy(outLocale, "en_US.UTF-8"); } else { tstrncpy(outLocale, locale, TD_LOCALE_LEN); - // printf("locale not configured, set to system default:%s", outLocale); + printf("locale not configured, set to system default:%s\n", outLocale); } // if user does not specify the charset, extract it from locale From 206887166c2a6698b87ade9c8474312a32345aab Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sun, 23 Apr 2023 10:53:28 +0800 Subject: [PATCH 050/109] fix: vnode errcode setting error --- source/dnode/vnode/src/vnd/vnodeSvr.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index b62bf27def..0dfde8f579 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -1348,6 +1348,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq code = terrno; goto _exit; } + terrno = 0; pSubmitTbData->uid = pSubmitTbData->pCreateTbReq->uid; // update uid if table exist for using below } } From 6fc364db199ada4a9f9b4410ed1a5bc2cd40faea Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 23 Apr 2023 13:53:55 +0800 Subject: [PATCH 051/109] fix(query): add null ptr check. --- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index fa8870835c..eb383df48d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -88,6 +88,10 @@ void getLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo, int64_t *blocks, double } void *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) { + if (pLoadInfo == NULL) { + return NULL; + } + for (int32_t i = 0; i < pLoadInfo->numOfStt; ++i) { pLoadInfo[i].currentLoadBlockIndex = 1; pLoadInfo[i].blockIndex[0] = -1; From da2ca889f28808569ef23b0c567b59416524741e Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 23 Apr 2023 14:42:24 +0800 Subject: [PATCH 052/109] fix: release resources if node open/start failed --- source/dnode/mgmt/node_mgmt/src/dmNodes.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/dnode/mgmt/node_mgmt/src/dmNodes.c b/source/dnode/mgmt/node_mgmt/src/dmNodes.c index 16931ab6df..8c1e6e745c 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmNodes.c +++ b/source/dnode/mgmt/node_mgmt/src/dmNodes.c @@ -107,6 +107,9 @@ static int32_t dmStartNodes(SDnode *pDnode) { dError("node:%s, failed to start since %s", pWrapper->name, terrstr()); return -1; } + if(ntype == VNODE) { + return -1; + } } dInfo("The daemon initialized successfully"); @@ -132,11 +135,14 @@ int32_t dmRunDnode(SDnode *pDnode) { int32_t count = 0; if (dmOpenNodes(pDnode) != 0) { dError("failed to open nodes since %s", terrstr()); + dmCloseNodes(pDnode); return -1; } if (dmStartNodes(pDnode) != 0) { dError("failed to start nodes since %s", terrstr()); + dmStopNodes(pDnode); + dmCloseNodes(pDnode); return -1; } From ebb1d83a3e21a7584fd09af37cd8773390075a66 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 23 Apr 2023 14:43:26 +0800 Subject: [PATCH 053/109] chore: remove ut code --- source/dnode/mgmt/node_mgmt/src/dmNodes.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/source/dnode/mgmt/node_mgmt/src/dmNodes.c b/source/dnode/mgmt/node_mgmt/src/dmNodes.c index 8c1e6e745c..df2a17a219 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmNodes.c +++ b/source/dnode/mgmt/node_mgmt/src/dmNodes.c @@ -107,9 +107,6 @@ static int32_t dmStartNodes(SDnode *pDnode) { dError("node:%s, failed to start since %s", pWrapper->name, terrstr()); return -1; } - if(ntype == VNODE) { - return -1; - } } dInfo("The daemon initialized successfully"); From f31e2b9cb420723c7900e88a48b01fdba9433ebe Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 23 Apr 2023 14:56:06 +0800 Subject: [PATCH 054/109] chore: more code --- source/dnode/mgmt/node_mgmt/src/dmNodes.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/mgmt/node_mgmt/src/dmNodes.c b/source/dnode/mgmt/node_mgmt/src/dmNodes.c index df2a17a219..19d5e06c5b 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmNodes.c +++ b/source/dnode/mgmt/node_mgmt/src/dmNodes.c @@ -138,6 +138,7 @@ int32_t dmRunDnode(SDnode *pDnode) { if (dmStartNodes(pDnode) != 0) { dError("failed to start nodes since %s", terrstr()); + dmSetStatus(pDnode, DND_STAT_STOPPED); dmStopNodes(pDnode); dmCloseNodes(pDnode); return -1; From 072eb2e8188f21240819814ced823e1701c4ec1b Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 23 Apr 2023 15:04:19 +0800 Subject: [PATCH 055/109] fix: taosbenchmark rest socket in stb section for main (#21020) --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 4838e97dd7..d9d2f12069 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG ffc2e6f + GIT_TAG 4378702 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 4291cfda21db3740bcb23faeaf7935734bc93a4e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 23 Apr 2023 07:18:23 +0000 Subject: [PATCH 056/109] fix filter err --- source/libs/index/src/indexFilter.c | 207 +++++++++++++++++++--------- tests/script/tsim/tag/bigint.sim | 11 ++ tests/script/tsim/tag/double.sim | 9 ++ tests/script/tsim/tag/int.sim | 10 ++ tests/script/tsim/tag/int_float.sim | 12 ++ tests/script/tsim/tag/tinyint.sim | 22 +++ 6 files changed, 207 insertions(+), 64 deletions(-) diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 23751cf26b..f1a32a578a 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -384,11 +384,11 @@ static int32_t sifInitOperParams(SIFParam **params, SOperatorNode *node, SIFCtx SIF_ERR_JRET(sifInitParam(node->pLeft, ¶mList[0], ctx)); if (nParam > 1) { - if (sifNeedConvertCond(node->pLeft, node->pRight)) { - SIF_ERR_JRET(sifInitParamValByCol(node->pLeft, node->pRight, ¶mList[1], ctx)); - } else { - SIF_ERR_JRET(sifInitParam(node->pRight, ¶mList[1], ctx)); - } + // if (sifNeedConvertCond(node->pLeft, node->pRight)) { + // SIF_ERR_JRET(sifInitParamValByCol(node->pLeft, node->pRight, ¶mList[1], ctx)); + // } else { + SIF_ERR_JRET(sifInitParam(node->pRight, ¶mList[1], ctx)); + // } // if (paramList[0].colValType == TSDB_DATA_TYPE_JSON && // ((SOperatorNode *)(node))->opType == OP_TYPE_JSON_CONTAINS) { // return TSDB_CODE_OUT_OF_MEMORY; @@ -474,65 +474,144 @@ static FORCE_INLINE FilterFunc sifGetFilterFunc(EIndexQueryType type, bool *reve } return NULL; } - -static void sifSetFltParam(SIFParam *left, SIFParam *right, SDataTypeBuf *typedata, SMetaFltParam *param) { - int8_t ltype = left->colValType, rtype = right->colValType; - if (!IS_VAR_DATA_TYPE(ltype) && IS_VAR_DATA_TYPE(rtype)) { - } else if (IS_VAR_DATA_TYPE(ltype) && !IS_VAR_DATA_TYPE(rtype)) { - return; - } else if (!IS_VAR_DATA_TYPE(ltype) && !IS_VAR_DATA_TYPE(rtype)) { - if (ltype == TSDB_DATA_TYPE_FLOAT) { - float f = 0; - SIF_DATA_CONVERT(rtype, right->condValue, f); - typedata->f = f; - param->val = &typedata->f; - } else if (ltype == TSDB_DATA_TYPE_DOUBLE) { - double d = 0; - SIF_DATA_CONVERT(rtype, right->condValue, d); - typedata->d = d; - param->val = &typedata->d; - } else if (ltype == TSDB_DATA_TYPE_BIGINT) { - int64_t i64 = 0; - SIF_DATA_CONVERT(rtype, right->condValue, i64); - typedata->i64 = i64; - param->val = &typedata->i64; - } else if (ltype == TSDB_DATA_TYPE_INT) { - int32_t i32 = 0; - SIF_DATA_CONVERT(rtype, right->condValue, i32); - typedata->i32 = i32; - param->val = &typedata->i32; - } else if (ltype == TSDB_DATA_TYPE_SMALLINT) { - int16_t i16 = 0; - SIF_DATA_CONVERT(rtype, right->condValue, i16); - typedata->i16 = i16; - param->val = &typedata->i16; - } else if (ltype == TSDB_DATA_TYPE_TINYINT) { - int8_t i8 = 0; - SIF_DATA_CONVERT(rtype, right->condValue, i8) - typedata->i8 = i8; - param->val = &typedata->i8; - } else if (ltype == TSDB_DATA_TYPE_UBIGINT) { - uint64_t u64 = 0; - SIF_DATA_CONVERT(rtype, right->condValue, u64); - typedata->u64 = u64; - param->val = &typedata->u64; - } else if (ltype == TSDB_DATA_TYPE_UINT) { - uint32_t u32 = 0; - SIF_DATA_CONVERT(rtype, right->condValue, u32); - typedata->u32 = u32; - param->val = &typedata->u32; - } else if (ltype == TSDB_DATA_TYPE_USMALLINT) { - uint16_t u16 = 0; - SIF_DATA_CONVERT(rtype, right->condValue, u16); - typedata->u16 = u16; - param->val = &typedata->u16; - } else if (ltype == TSDB_DATA_TYPE_UTINYINT) { - uint8_t u8 = 0; - SIF_DATA_CONVERT(rtype, right->condValue, u8); - typedata->u8 = u8; - param->val = &typedata->u8; - } +int32_t sifStr2num(char *buf, int8_t type, void *val) { + if (type == TSDB_DATA_TYPE_TINYINT) { + int8_t v = 0; + if (1 != sscanf(buf, "%hhd", &v)) return -1; + *(int8_t *)val = v; + } else if (type == TSDB_DATA_TYPE_SMALLINT) { + int16_t v = 0; + if (1 != sscanf(buf, "%hd", &v)) return -1; + *(int16_t *)val = v; + } else if (type == TSDB_DATA_TYPE_INT) { + int32_t v = 0; + if (1 != sscanf(buf, "%d", &v)) return -1; + *(int32_t *)val = v; + } else if (type == TSDB_DATA_TYPE_BIGINT) { + int64_t v = 0; + if (1 != sscanf(buf, "%" PRId64 "", &v)) return -1; + *(int64_t *)val = v; + } else if (type == TSDB_DATA_TYPE_FLOAT) { + float v = 0; + if (1 != sscanf(buf, "%f", &v)) return -1; + *(float *)val = v; + } else if (type == TSDB_DATA_TYPE_DOUBLE) { + double v = 0; + if (1 != sscanf(buf, "%lf", &v)) return -1; + *(double *)val = v; + } else if (type == TSDB_DATA_TYPE_UBIGINT) { + uint64_t v = 0; + if (1 != sscanf(buf, "%" PRIu64 "", &v)) return -1; + *(uint64_t *)val = v; + } else if (type == TSDB_DATA_TYPE_UINT || type == TSDB_DATA_TYPE_UTINYINT || type == TSDB_DATA_TYPE_USMALLINT) { + uint32_t v = 0; + if (1 != sscanf(buf, "%u", &v)) return -1; + *(uint32_t *)val = v; + } else { + return -1; } + return 0; +} + +static int32_t sifSetFltParam(SIFParam *left, SIFParam *right, SDataTypeBuf *typedata, SMetaFltParam *param) { + int32_t code = 0; + int8_t ltype = left->colValType, rtype = right->colValType; + if (!IS_NUMERIC_TYPE(ltype) || !((IS_NUMERIC_TYPE(rtype)) || rtype == TSDB_DATA_TYPE_VARCHAR)) { + return -1; + } + if (ltype == TSDB_DATA_TYPE_FLOAT) { + float f = 0; + if (IS_NUMERIC_TYPE(rtype)) { + SIF_DATA_CONVERT(rtype, right->condValue, f); + } else { + SIF_ERR_RET(sifStr2num(right->condValue + VARSTR_HEADER_SIZE, TSDB_DATA_TYPE_FLOAT, &f)); + } + typedata->f = f; + param->val = &typedata->f; + } else if (ltype == TSDB_DATA_TYPE_DOUBLE) { + double d = 0; + if (IS_NUMERIC_TYPE(rtype)) { + SIF_DATA_CONVERT(rtype, right->condValue, d); + } else { + SIF_ERR_RET(sifStr2num(right->condValue + VARSTR_HEADER_SIZE, TSDB_DATA_TYPE_DOUBLE, &d)); + } + typedata->d = d; + param->val = &typedata->d; + } else if (ltype == TSDB_DATA_TYPE_BIGINT) { + int64_t i64 = 0; + if (IS_NUMERIC_TYPE(rtype)) { + SIF_DATA_CONVERT(rtype, right->condValue, i64); + } else { + SIF_ERR_RET(sifStr2num(right->condValue + VARSTR_HEADER_SIZE, TSDB_DATA_TYPE_BIGINT, &i64)); + } + typedata->i64 = i64; + param->val = &typedata->i64; + } else if (ltype == TSDB_DATA_TYPE_INT) { + int32_t i32 = 0; + if (IS_NUMERIC_TYPE(rtype)) { + SIF_DATA_CONVERT(rtype, right->condValue, i32); + } else { + SIF_ERR_RET(sifStr2num(right->condValue + VARSTR_HEADER_SIZE, TSDB_DATA_TYPE_INT, &i32)); + } + typedata->i32 = i32; + param->val = &typedata->i32; + } else if (ltype == TSDB_DATA_TYPE_SMALLINT) { + int16_t i16 = 0; + if (IS_NUMERIC_TYPE(rtype)) { + SIF_DATA_CONVERT(rtype, right->condValue, i16); + } else { + SIF_ERR_RET(sifStr2num(right->condValue + VARSTR_HEADER_SIZE, TSDB_DATA_TYPE_SMALLINT, &i16)); + } + + typedata->i16 = i16; + param->val = &typedata->i16; + } else if (ltype == TSDB_DATA_TYPE_TINYINT) { + int8_t i8 = 0; + if (IS_NUMERIC_TYPE(rtype)) { + SIF_DATA_CONVERT(rtype, right->condValue, i8); + } else { + SIF_ERR_RET(sifStr2num(right->condValue + VARSTR_HEADER_SIZE, TSDB_DATA_TYPE_TINYINT, &i8)); + } + typedata->i8 = i8; + param->val = &typedata->i8; + } else if (ltype == TSDB_DATA_TYPE_UBIGINT) { + uint64_t u64 = 0; + if (IS_NUMERIC_TYPE(rtype)) { + SIF_DATA_CONVERT(rtype, right->condValue, u64); + } else { + SIF_ERR_RET(sifStr2num(right->condValue + VARSTR_HEADER_SIZE, TSDB_DATA_TYPE_UBIGINT, &u64)); + } + typedata->u64 = u64; + param->val = &typedata->u64; + } else if (ltype == TSDB_DATA_TYPE_UINT) { + uint32_t u32 = 0; + if (IS_NUMERIC_TYPE(rtype)) { + SIF_DATA_CONVERT(rtype, right->condValue, u32); + } else { + SIF_ERR_RET(sifStr2num(right->condValue + VARSTR_HEADER_SIZE, TSDB_DATA_TYPE_UINT, &u32)); + } + typedata->u32 = u32; + param->val = &typedata->u32; + } else if (ltype == TSDB_DATA_TYPE_USMALLINT) { + uint16_t u16 = 0; + if (IS_NUMERIC_TYPE(rtype)) { + SIF_DATA_CONVERT(rtype, right->condValue, u16); + } else { + SIF_ERR_RET(sifStr2num(right->condValue + VARSTR_HEADER_SIZE, TSDB_DATA_TYPE_USMALLINT, &u16)); + } + typedata->u16 = u16; + param->val = &typedata->u16; + } else if (ltype == TSDB_DATA_TYPE_UTINYINT) { + uint8_t u8 = 0; + if (IS_NUMERIC_TYPE(rtype)) { + SIF_DATA_CONVERT(rtype, right->condValue, u8); + } else { + SIF_ERR_RET(sifStr2num(right->condValue + VARSTR_HEADER_SIZE, TSDB_DATA_TYPE_UTINYINT, &u8)); + } + typedata->u8 = u8; + param->val = &typedata->u8; + } + return 0; } static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) { int ret = 0; @@ -573,7 +652,7 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP param.val = buf; } } else { - sifSetFltParam(left, right, &typedata, ¶m); + if (sifSetFltParam(left, right, &typedata, ¶m) != 0) return -1; } ret = metaFilterTableIds(arg->metaEx, ¶m, output->result); } diff --git a/tests/script/tsim/tag/bigint.sim b/tests/script/tsim/tag/bigint.sim index 26a5addf6a..34fcc09411 100644 --- a/tests/script/tsim/tag/bigint.sim +++ b/tests/script/tsim/tag/bigint.sim @@ -123,6 +123,17 @@ sql select * from $mt where tgcol = 1 if $rows != 100 then return -1 endi + +sql select * from $mt where tgcol = '1' +if $rows != 100 then + return -1 +endi + +sql select * from $mt where tgcol = "1" +if $rows != 100 then + return -1 +endi + sql select * from $mt where tgcol <> 1 if $rows != 100 then return -1 diff --git a/tests/script/tsim/tag/double.sim b/tests/script/tsim/tag/double.sim index fbdf973337..acc026c13d 100644 --- a/tests/script/tsim/tag/double.sim +++ b/tests/script/tsim/tag/double.sim @@ -123,6 +123,15 @@ sql select * from $mt where tgcol = 1 if $rows != 100 then return -1 endi + +sql select * from $mt where tgcol = '1'; +if $rows != 100 then + return -1 +endi +sql select * from $mt where tgcol = "1.0" +if $rows != 100 then + return -1 +endi sql select * from $mt where tgcol <> 1 if $rows != 100 then return -1 diff --git a/tests/script/tsim/tag/int.sim b/tests/script/tsim/tag/int.sim index ac8d31db3b..084b7e74d7 100644 --- a/tests/script/tsim/tag/int.sim +++ b/tests/script/tsim/tag/int.sim @@ -123,6 +123,16 @@ sql select * from $mt where tgcol = 1 if $rows != 100 then return -1 endi + +sql select * from $mt where tgcol = '1' +if $rows != 100 then + return -1 +endi + +sql select * from $mt where tgcol = "1"; +if $rows != 100 then + return -1 +endi sql select * from $mt where tgcol <> 1 if $rows != 100 then return -1 diff --git a/tests/script/tsim/tag/int_float.sim b/tests/script/tsim/tag/int_float.sim index 009629aac9..3034f8b64e 100644 --- a/tests/script/tsim/tag/int_float.sim +++ b/tests/script/tsim/tag/int_float.sim @@ -85,10 +85,22 @@ sql select * from $mt where tgcol <> 1 if $rows != 100 then return -1 endi + sql select * from $mt where tgcol = 1 if $rows != 100 then return -1 endi + +sql select * from $mt where tgcol = '1' +if $rows != 100 then + return -1 +endi + +sql select * from $mt where tgcol = "1" +if $rows != 100 then + return -1 +endi + sql select * from $mt where tgcol <> 1 if $rows != 100 then return -1 diff --git a/tests/script/tsim/tag/tinyint.sim b/tests/script/tsim/tag/tinyint.sim index 8560def34c..89b0134bb3 100644 --- a/tests/script/tsim/tag/tinyint.sim +++ b/tests/script/tsim/tag/tinyint.sim @@ -115,14 +115,36 @@ sql select * from $mt where tgcol = 0 if $rows != 100 then return -1 endi + +sql select * from $mt where tgcol = '0' +if $rows != 100 then + return -1 +endi + +sql select * from $mt where tgcol = "0" +if $rows != 100 then + return -1 +endi sql select * from $mt where tgcol <> 0 if $rows != 100 then return -1 endi + sql select * from $mt where tgcol = 1 if $rows != 100 then return -1 endi +sql select * from $mt where tgcol = "1" +if $rows != 100 then + return -1 +endi + +sql select * from $mt where tgcol = '1' +if $rows != 100 then + return -1 +endi + + sql select * from $mt where tgcol <> 1 if $rows != 100 then return -1 From 03b661061afff591a59ce5a503abe56f621ea2f0 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 23 Apr 2023 08:12:36 +0000 Subject: [PATCH 057/109] fix filter err --- source/libs/index/src/indexFilter.c | 88 +++++++++++++++-------------- 1 file changed, 45 insertions(+), 43 deletions(-) diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index f1a32a578a..7b49153fab 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -474,41 +474,41 @@ static FORCE_INLINE FilterFunc sifGetFilterFunc(EIndexQueryType type, bool *reve } return NULL; } -int32_t sifStr2num(char *buf, int8_t type, void *val) { - if (type == TSDB_DATA_TYPE_TINYINT) { - int8_t v = 0; - if (1 != sscanf(buf, "%hhd", &v)) return -1; - *(int8_t *)val = v; - } else if (type == TSDB_DATA_TYPE_SMALLINT) { - int16_t v = 0; - if (1 != sscanf(buf, "%hd", &v)) return -1; - *(int16_t *)val = v; - } else if (type == TSDB_DATA_TYPE_INT) { - int32_t v = 0; - if (1 != sscanf(buf, "%d", &v)) return -1; - *(int32_t *)val = v; - } else if (type == TSDB_DATA_TYPE_BIGINT) { +int32_t sifStr2Num(char *buf, int32_t len, int8_t type, void *val) { + if (IS_SIGNED_NUMERIC_TYPE(type)) { int64_t v = 0; - if (1 != sscanf(buf, "%" PRId64 "", &v)) return -1; - *(int64_t *)val = v; - } else if (type == TSDB_DATA_TYPE_FLOAT) { - float v = 0; - if (1 != sscanf(buf, "%f", &v)) return -1; - *(float *)val = v; - } else if (type == TSDB_DATA_TYPE_DOUBLE) { - double v = 0; - if (1 != sscanf(buf, "%lf", &v)) return -1; - *(double *)val = v; - } else if (type == TSDB_DATA_TYPE_UBIGINT) { + if (0 != toInteger(buf, len, 10, &v)) { + return -1; + } + if (type == TSDB_DATA_TYPE_BIGINT) { + *(int64_t *)val = v; + } else if (type == TSDB_DATA_TYPE_INT) { + *(int32_t *)val = v; + } else if (type == TSDB_DATA_TYPE_TINYINT) { + *(int8_t *)val = v; + } else if (type == TSDB_DATA_TYPE_SMALLINT) { + *(int16_t *)val = v; + } + } else if (IS_FLOAT_TYPE(type)) { + if (type == TSDB_DATA_TYPE_FLOAT) { + *(float *)val = taosStr2Float(buf, NULL); + } else { + *(double *)val = taosStr2Double(buf, NULL); + } + } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { uint64_t v = 0; - if (1 != sscanf(buf, "%" PRIu64 "", &v)) return -1; - *(uint64_t *)val = v; - } else if (type == TSDB_DATA_TYPE_UINT || type == TSDB_DATA_TYPE_UTINYINT || type == TSDB_DATA_TYPE_USMALLINT) { - uint32_t v = 0; - if (1 != sscanf(buf, "%u", &v)) return -1; - *(uint32_t *)val = v; - } else { - return -1; + if (0 != toUInteger(buf, len, 10, &v)) { + return -1; + } + if (type == TSDB_DATA_TYPE_UBIGINT) { + *(uint64_t *)val = v; + } else if (type == TSDB_DATA_TYPE_UINT) { + *(uint32_t *)val = v; + } else if (type == TSDB_DATA_TYPE_UTINYINT) { + *(uint8_t *)val = v; + } else if (type == TSDB_DATA_TYPE_USMALLINT) { + *(uint16_t *)val = v; + } } return 0; } @@ -524,7 +524,7 @@ static int32_t sifSetFltParam(SIFParam *left, SIFParam *right, SDataTypeBuf *typ if (IS_NUMERIC_TYPE(rtype)) { SIF_DATA_CONVERT(rtype, right->condValue, f); } else { - SIF_ERR_RET(sifStr2num(right->condValue + VARSTR_HEADER_SIZE, TSDB_DATA_TYPE_FLOAT, &f)); + SIF_ERR_RET(sifStr2Num(varDataVal(right->condValue), varDataLen(right->condValue), TSDB_DATA_TYPE_FLOAT, &f)); } typedata->f = f; param->val = &typedata->f; @@ -533,7 +533,7 @@ static int32_t sifSetFltParam(SIFParam *left, SIFParam *right, SDataTypeBuf *typ if (IS_NUMERIC_TYPE(rtype)) { SIF_DATA_CONVERT(rtype, right->condValue, d); } else { - SIF_ERR_RET(sifStr2num(right->condValue + VARSTR_HEADER_SIZE, TSDB_DATA_TYPE_DOUBLE, &d)); + SIF_ERR_RET(sifStr2Num(varDataVal(right->condValue), varDataLen(right->condValue), TSDB_DATA_TYPE_DOUBLE, &d)); } typedata->d = d; param->val = &typedata->d; @@ -542,7 +542,7 @@ static int32_t sifSetFltParam(SIFParam *left, SIFParam *right, SDataTypeBuf *typ if (IS_NUMERIC_TYPE(rtype)) { SIF_DATA_CONVERT(rtype, right->condValue, i64); } else { - SIF_ERR_RET(sifStr2num(right->condValue + VARSTR_HEADER_SIZE, TSDB_DATA_TYPE_BIGINT, &i64)); + SIF_ERR_RET(sifStr2Num(varDataVal(right->condValue), varDataLen(right->condValue), TSDB_DATA_TYPE_BIGINT, &i64)); } typedata->i64 = i64; param->val = &typedata->i64; @@ -551,7 +551,7 @@ static int32_t sifSetFltParam(SIFParam *left, SIFParam *right, SDataTypeBuf *typ if (IS_NUMERIC_TYPE(rtype)) { SIF_DATA_CONVERT(rtype, right->condValue, i32); } else { - SIF_ERR_RET(sifStr2num(right->condValue + VARSTR_HEADER_SIZE, TSDB_DATA_TYPE_INT, &i32)); + SIF_ERR_RET(sifStr2Num(varDataVal(right->condValue), varDataLen(right->condValue), TSDB_DATA_TYPE_INT, &i32)); } typedata->i32 = i32; param->val = &typedata->i32; @@ -560,7 +560,8 @@ static int32_t sifSetFltParam(SIFParam *left, SIFParam *right, SDataTypeBuf *typ if (IS_NUMERIC_TYPE(rtype)) { SIF_DATA_CONVERT(rtype, right->condValue, i16); } else { - SIF_ERR_RET(sifStr2num(right->condValue + VARSTR_HEADER_SIZE, TSDB_DATA_TYPE_SMALLINT, &i16)); + SIF_ERR_RET( + sifStr2Num(varDataVal(right->condValue), varDataLen(right->condValue), TSDB_DATA_TYPE_SMALLINT, &i16)); } typedata->i16 = i16; @@ -570,7 +571,7 @@ static int32_t sifSetFltParam(SIFParam *left, SIFParam *right, SDataTypeBuf *typ if (IS_NUMERIC_TYPE(rtype)) { SIF_DATA_CONVERT(rtype, right->condValue, i8); } else { - SIF_ERR_RET(sifStr2num(right->condValue + VARSTR_HEADER_SIZE, TSDB_DATA_TYPE_TINYINT, &i8)); + SIF_ERR_RET(sifStr2Num(varDataVal(right->condValue), varDataLen(right->condValue), TSDB_DATA_TYPE_TINYINT, &i8)); } typedata->i8 = i8; param->val = &typedata->i8; @@ -579,7 +580,7 @@ static int32_t sifSetFltParam(SIFParam *left, SIFParam *right, SDataTypeBuf *typ if (IS_NUMERIC_TYPE(rtype)) { SIF_DATA_CONVERT(rtype, right->condValue, u64); } else { - SIF_ERR_RET(sifStr2num(right->condValue + VARSTR_HEADER_SIZE, TSDB_DATA_TYPE_UBIGINT, &u64)); + SIF_ERR_RET(sifStr2Num(varDataVal(right->condValue), varDataLen(right->condValue), TSDB_DATA_TYPE_UBIGINT, &u64)); } typedata->u64 = u64; param->val = &typedata->u64; @@ -588,7 +589,7 @@ static int32_t sifSetFltParam(SIFParam *left, SIFParam *right, SDataTypeBuf *typ if (IS_NUMERIC_TYPE(rtype)) { SIF_DATA_CONVERT(rtype, right->condValue, u32); } else { - SIF_ERR_RET(sifStr2num(right->condValue + VARSTR_HEADER_SIZE, TSDB_DATA_TYPE_UINT, &u32)); + SIF_ERR_RET(sifStr2Num(varDataVal(right->condValue), varDataLen(right->condValue), TSDB_DATA_TYPE_UINT, &u32)); } typedata->u32 = u32; param->val = &typedata->u32; @@ -597,7 +598,8 @@ static int32_t sifSetFltParam(SIFParam *left, SIFParam *right, SDataTypeBuf *typ if (IS_NUMERIC_TYPE(rtype)) { SIF_DATA_CONVERT(rtype, right->condValue, u16); } else { - SIF_ERR_RET(sifStr2num(right->condValue + VARSTR_HEADER_SIZE, TSDB_DATA_TYPE_USMALLINT, &u16)); + SIF_ERR_RET( + sifStr2Num(varDataVal(right->condValue), varDataLen(right->condValue), TSDB_DATA_TYPE_USMALLINT, &u16)); } typedata->u16 = u16; param->val = &typedata->u16; @@ -606,7 +608,7 @@ static int32_t sifSetFltParam(SIFParam *left, SIFParam *right, SDataTypeBuf *typ if (IS_NUMERIC_TYPE(rtype)) { SIF_DATA_CONVERT(rtype, right->condValue, u8); } else { - SIF_ERR_RET(sifStr2num(right->condValue + VARSTR_HEADER_SIZE, TSDB_DATA_TYPE_UTINYINT, &u8)); + SIF_ERR_RET(sifStr2Num(varDataVal(right->condValue), varDataLen(right->condValue), TSDB_DATA_TYPE_UTINYINT, &u8)); } typedata->u8 = u8; param->val = &typedata->u8; From 2699a0144462a616fe039fa5aa139a282757c373 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 23 Apr 2023 08:13:27 +0000 Subject: [PATCH 058/109] fix filter err --- source/libs/index/src/indexFilter.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 7b49153fab..02ed0d2d05 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -475,6 +475,7 @@ static FORCE_INLINE FilterFunc sifGetFilterFunc(EIndexQueryType type, bool *reve return NULL; } int32_t sifStr2Num(char *buf, int32_t len, int8_t type, void *val) { + // signed/unsigned/float if (IS_SIGNED_NUMERIC_TYPE(type)) { int64_t v = 0; if (0 != toInteger(buf, len, 10, &v)) { @@ -509,6 +510,8 @@ int32_t sifStr2Num(char *buf, int32_t len, int8_t type, void *val) { } else if (type == TSDB_DATA_TYPE_USMALLINT) { *(uint16_t *)val = v; } + } else { + return -1; } return 0; } From 4bb7a25fc7491f5f1722b8994c646f94fa3c6ec0 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sun, 23 Apr 2023 16:30:28 +0800 Subject: [PATCH 059/109] feat: subtable level privilege --- include/common/tmsg.h | 1 + source/common/src/tmsg.c | 9 +- source/dnode/mnode/impl/src/mndUser.c | 2 + source/libs/catalog/src/ctgUtil.c | 9 +- source/libs/nodes/src/nodesUtilFuncs.c | 8 +- source/libs/parser/src/parAuthenticator.c | 15 +- source/libs/parser/src/parTranslater.c | 4 +- tests/script/tsim/user/privilege_table.sim | 280 +++++++++++++++++++++ 8 files changed, 307 insertions(+), 21 deletions(-) create mode 100644 tests/script/tsim/user/privilege_table.sim diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 02c097b8d0..df76edffc9 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -689,6 +689,7 @@ typedef struct { int32_t tSerializeSAlterUserReq(void* buf, int32_t bufLen, SAlterUserReq* pReq); int32_t tDeserializeSAlterUserReq(void* buf, int32_t bufLen, SAlterUserReq* pReq); +void tFreeSAlterUserReq(SAlterUserReq* pReq); typedef struct { char user[TSDB_USER_LEN]; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index d9802244b7..189fa1326f 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1409,6 +1409,8 @@ int32_t tDeserializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq return 0; } +void tFreeSAlterUserReq(SAlterUserReq *pReq) { taosMemoryFreeClear(pReq->tagCond); } + int32_t tSerializeSGetUserAuthReq(void *buf, int32_t bufLen, SGetUserAuthReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -1635,6 +1637,7 @@ int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRs int32_t ref = 0; if (tDecodeI32(pDecoder, &ref) < 0) return -1; taosHashPut(pRsp->useDbs, key, strlen(key), &ref, sizeof(ref)); + taosMemoryFree(key); } } @@ -1831,7 +1834,6 @@ int32_t tSerializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pReq if (tEncodeCStr(&encoder, pReq->pComment) < 0) return -1; } - if (tEncodeI8(&encoder, pReq->orReplace) < 0) return -1; tEndEncode(&encoder); @@ -1876,7 +1878,6 @@ int32_t tDeserializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pR if (tDecodeCStrTo(&decoder, pReq->pComment) < 0) return -1; } - if (!tDecodeIsEnd(&decoder)) { if (tDecodeI8(&decoder, &pReq->orReplace) < 0) return -1; } else { @@ -2053,12 +2054,12 @@ int32_t tDeserializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp if (pRsp->pFuncExtraInfos == NULL) return -1; if (tDecodeIsEnd(&decoder)) { for (int32_t i = 0; i < pRsp->numOfFuncs; ++i) { - SFuncExtraInfo extraInfo = { 0 }; + SFuncExtraInfo extraInfo = {0}; taosArrayPush(pRsp->pFuncExtraInfos, &extraInfo); } } else { for (int32_t i = 0; i < pRsp->numOfFuncs; ++i) { - SFuncExtraInfo extraInfo = { 0 }; + SFuncExtraInfo extraInfo = {0}; if (tDecodeI32(&decoder, &extraInfo.funcVersion) < 0) return -1; if (tDecodeI64(&decoder, &extraInfo.funcCreatedTime) < 0) return -1; taosArrayPush(pRsp->pFuncExtraInfos, &extraInfo); diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index d08227927a..523753d7c6 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -390,6 +390,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &ref, _OVER); taosHashPut(pUser->useDbs, key, keyLen, &ref, sizeof(ref)); + taosMemoryFree(key); } } @@ -956,6 +957,7 @@ _OVER: mError("user:%s, failed to alter since %s", alterReq.user, terrstr()); } + tFreeSAlterUserReq(&alterReq); mndReleaseUser(pMnode, pOperUser); mndReleaseUser(pMnode, pUser); mndUserFreeObj(&newUser); diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 7ef2e34d1e..62896e4307 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -486,12 +486,12 @@ void ctgFreeBatchHash(void* hash) { taosMemoryFreeClear(pRes->pRes); } -void ctgFreeJsonTagVal(void *val) { +void ctgFreeJsonTagVal(void* val) { if (NULL == val) { return; } - STagVal *pVal = (STagVal *)val; + STagVal* pVal = (STagVal*)val; if (TSDB_DATA_TYPE_JSON == pVal->type) { taosMemoryFree(pVal->pData); @@ -545,6 +545,7 @@ void ctgFreeTaskRes(CTG_TASK_TYPE type, void** pRes) { taosArrayDestroy(*pRes); } *pRes = NULL; + break; } case CTG_TASK_GET_TB_META_BATCH: { SArray* pArray = (SArray*)*pRes; @@ -1363,7 +1364,7 @@ int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { SGetUserAuthRsp* pInfo = &req->authInfo; SHashObj* pTbs = (AUTH_TYPE_READ == req->singleType) ? pInfo->readTbs : pInfo->writeTbs; char* stbName = NULL; - + char tbFName[TSDB_TABLE_FNAME_LEN]; char dbFName[TSDB_DB_FNAME_LEN]; tNameExtractFullName(&req->pRawReq->tbName, tbFName); @@ -1396,7 +1397,7 @@ int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { SCtgTbMetaCtx ctx = {0}; ctx.pName = (SName*)&req->pRawReq->tbName; ctx.flag = CTG_FLAG_UNKNOWN_STB | CTG_FLAG_SYNC_OP; - + CTG_ERR_RET(ctgGetTbMeta(pCtg, req->pConn, &ctx, &pMeta)); } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 3f571e22ae..422a196c50 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -953,8 +953,12 @@ void nodesDestroyNode(SNode* pNode) { break; case QUERY_NODE_SPLIT_VGROUP_STMT: // no pointer field case QUERY_NODE_SYNCDB_STMT: // no pointer field - case QUERY_NODE_GRANT_STMT: // no pointer field - case QUERY_NODE_REVOKE_STMT: // no pointer field + break; + case QUERY_NODE_GRANT_STMT: + nodesDestroyNode(((SGrantStmt*)pNode)->pTagCond); + break; + case QUERY_NODE_REVOKE_STMT: + nodesDestroyNode(((SRevokeStmt*)pNode)->pTagCond); break; case QUERY_NODE_SHOW_DNODES_STMT: case QUERY_NODE_SHOW_MNODES_STMT: diff --git a/source/libs/parser/src/parAuthenticator.c b/source/libs/parser/src/parAuthenticator.c index b06d48a690..d182f5bd73 100644 --- a/source/libs/parser/src/parAuthenticator.c +++ b/source/libs/parser/src/parAuthenticator.c @@ -70,7 +70,7 @@ static EDealRes authSubquery(SAuthCxt* pCxt, SNode* pStmt) { return TSDB_CODE_SUCCESS == authQuery(pCxt, pStmt) ? DEAL_RES_CONTINUE : DEAL_RES_ERROR; } -static int32_t mergeStableTagCond(SNode** pWhere, SNode** pTagCond) { +static int32_t mergeStableTagCond(SNode** pWhere, SNode* pTagCond) { SLogicConditionNode* pLogicCond = (SLogicConditionNode*)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); if (NULL == pLogicCond) { return TSDB_CODE_OUT_OF_MEMORY; @@ -78,7 +78,7 @@ static int32_t mergeStableTagCond(SNode** pWhere, SNode** pTagCond) { pLogicCond->node.resType.type = TSDB_DATA_TYPE_BOOL; pLogicCond->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; pLogicCond->condType = LOGIC_COND_TYPE_AND; - int32_t code = nodesListMakeStrictAppend(&pLogicCond->pParameterList, *pTagCond); + int32_t code = nodesListMakeStrictAppend(&pLogicCond->pParameterList, pTagCond); if (TSDB_CODE_SUCCESS == code) { code = nodesListMakeAppend(&pLogicCond->pParameterList, *pWhere); } @@ -91,22 +91,17 @@ static int32_t mergeStableTagCond(SNode** pWhere, SNode** pTagCond) { } static int32_t appendStableTagCond(SNode** pWhere, SNode* pTagCond) { - SNode* pTagCondCopy = nodesCloneNode(pTagCond); - if (NULL == pTagCondCopy) { - return TSDB_CODE_OUT_OF_MEMORY; - } - if (NULL == *pWhere) { - *pWhere = pTagCondCopy; + *pWhere = pTagCond; return TSDB_CODE_SUCCESS; } if (QUERY_NODE_LOGIC_CONDITION == nodeType(*pWhere) && LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)*pWhere)->condType) { - return nodesListStrictAppend(((SLogicConditionNode*)*pWhere)->pParameterList, pTagCondCopy); + return nodesListStrictAppend(((SLogicConditionNode*)*pWhere)->pParameterList, pTagCond); } - return mergeStableTagCond(pWhere, &pTagCondCopy); + return mergeStableTagCond(pWhere, pTagCond); } static EDealRes authSelectImpl(SNode* pNode, void* pContext) { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index b44c36dde1..200206755b 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1310,7 +1310,8 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { } static EDealRes haveVectorFunction(SNode* pNode, void* pContext) { - if (isAggFunc(pNode) || isIndefiniteRowsFunc(pNode) || isWindowPseudoColumnFunc(pNode) || isInterpPseudoColumnFunc(pNode)) { + if (isAggFunc(pNode) || isIndefiniteRowsFunc(pNode) || isWindowPseudoColumnFunc(pNode) || + isInterpPseudoColumnFunc(pNode)) { *((bool*)pContext) = true; return DEAL_RES_END; } @@ -6617,6 +6618,7 @@ static int32_t translateGrant(STranslateContext* pCxt, SGrantStmt* pStmt) { if (TSDB_CODE_SUCCESS == code) { code = buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &req); } + tFreeSAlterUserReq(&req); return code; } diff --git a/tests/script/tsim/user/privilege_table.sim b/tests/script/tsim/user/privilege_table.sim new file mode 100644 index 0000000000..5256cdd21d --- /dev/null +++ b/tests/script/tsim/user/privilege_table.sim @@ -0,0 +1,280 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print =============== init env +sql drop database if exists test; +sql create database test vgroups 1; +sql use test; +sql create stable st1(ts timestamp, i int) tags(id int, loc varchar(20)); +sql create table st1s1 using st1 tags(1, 'beijing'); +sql create table st1s2 using st1 tags(2, 'shanghai'); +sql insert into st1s1 values(now, 1) st1s2 values(now, 2); +sql create stable st2(ts timestamp, i int) tags(id int, loc varchar(20)); +sql create table st2s1 using st2 tags(1, 'beijing'); +sql create table st2s2 using st2 tags(2, 'shanghai'); +sql insert into st2s1 values(now, 1) st2s2 values(now, 2); +sql create user wxy pass 'taosdata'; + +print =============== case 1: database unauthorized and table unauthorized +sql close +sql connect wxy + +sql reset query cache; +sql_error select * from test.st1; +sql_error insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20); +sql_error select * from test.st2; +sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); + + +print =============== case 2: database unauthorized and table read privilege +sql close +sql connect + +sql grant read on test.st1 to wxy; + +sql close +sql connect wxy + +sql reset query cache; +sql select * from test.st1; +if $rows != 2 then + return -1 +endi +sql_error insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20); +sql_error select * from test.st2; +sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); + +print =============== case 3: database unauthorized and table read privilege with condition +sql close +sql connect + +sql revoke read on test.st1 from wxy; +sql grant read on test.st1 with id = 1 to wxy; + +sql close +sql connect wxy + +sql reset query cache; +sql select * from test.st1; +if $rows != 1 then + return -1 +endi +sql_error insert into test.st1s1 values(now, 10); +sql_error insert into test.st1s2 values(now, 20); +sql_error select * from test.st2; +sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); + +print =============== case 4: database unauthorized and table write privilege +sql close +sql connect + +sql revoke read on test.st1 with id = 1 from wxy; +sql grant write on test.st1 to wxy; + +sql close +sql connect wxy + +sql reset query cache; +sql_error select tbname, * from test.st1; +sql insert into test.st1s1 values(now, 10); +sql insert into test.st1s2 values(now, 20); +sql_error select * from test.st2; +sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); + +print =============== case 5: database unauthorized and table write privilege with condition +sql close +sql connect + +sql revoke write on test.st1 from wxy; +sql grant write on test.st1 with id = 1 to wxy; + +sql close +sql connect wxy + +sql reset query cache; +sql_error select tbname, * from test.st1; +sql insert into test.st1s1 values(now, 10); +sql_error insert into test.st1s2 values(now, 20); +sql_error select * from test.st2; +sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); + +print =============== case 6: database read privilege and table unauthorized +sql close +sql connect + +sql revoke write on test.st1 with id = 1 from wxy; +sql grant read on test.* to wxy; + +sql close +sql connect wxy + +sql reset query cache; +sql select * from test.st1; +if $rows != 5 then + return -1 +endi +sql_error insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20); +sql select * from test.st2; +if $rows != 2 then + return -1 +endi +sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); + +print =============== case 7: database read privilege and table read privilege +sql close +sql connect + +sql grant read on test.st1 to wxy; + +sql close +sql connect wxy + +sql reset query cache; +sql select * from test.st1; +if $rows != 2 then + return -1 +endi +sql_error insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20); +sql select * from test.st2; +if $rows != 2 then + return -1 +endi +sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); + +print =============== case 8: database read privilege and table read privilege with condition +sql close +sql connect + +sql revoke read on test.st1 from wxy; +sql grant read on test.st1 with id = 1 to wxy; + +sql close +sql connect wxy + +sql select * from test.st1; +if $rows != 1 then + return -1 +endi +sql_error insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20); +sql select * from test.st2; +if $rows != 2 then + return -1 +endi +sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); + +print =============== case 9: database read privilege and table write privilege +sql close +sql connect + +sql revoke read on test.st1 with id = 1 from wxy; +sql grant write on test.st1 to wxy; + +sql close +sql connect wxy + +sql select * from test.st1; +if $rows != 2 then + return -1 +endi +sql insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20); +sql select * from test.st2; +if $rows != 2 then + return -1 +endi +sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); + +print =============== case 10: database read privilege and table write privilege with condition +sql close +sql connect + +sql revoke write on test.st1 from wxy; +sql grant write on test.st1 with id = 1 to wxy; + +sql close +sql connect wxy + +sql select * from test.st1; +sql insert into test.st1s1 values(now, 10); +sql_error insert into test.st1s2 values(now, 20); +sql select * from test.st2; +sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); + +print =============== case 11: database write privilege and table unauthorized +sql close +sql connect + +sql revoke read on test.* from wxy; +sql revoke write on test.st1 with id = 1 from wxy; +sql grant write on test.* to wxy; + +sql close +sql connect wxy + +sql_error select * from test.st1; +sql insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20); +sql_error select * from test.st2; +sql insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); + +print =============== case 12: database write privilege and table read privilege +sql close +sql connect + +sql grant read on test.st1 to wxy; + +sql close +sql connect wxy + +sql select * from test.st1; +sql insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20); +sql_error select * from test.st2; +sql insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); + +print =============== case 13: database write privilege and table read privilege with condition +sql close +sql connect + +sql revoke read on test.st1 from wxy; +sql grant read on test.st1 with id = 1 to wxy; + +sql close +sql connect wxy + +sql select * from test.st1; +sql insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20); +sql_error select * from test.st2; +sql insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); + +print =============== case 14: database write privilege and table write privilege +sql close +sql connect + +sql revoke read on test.st1 with id = 1 from wxy; +sql grant write on test.st1 to wxy; + +sql close +sql connect wxy + +sql_error select * from test.st1; +sql insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20); +sql_error select * from test.st2; +sql insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); + +print =============== case 15: database write privilege and table write privilege with condition +sql close +sql connect + +sql revoke write on test.st1 from wxy; +sql grant write on test.st1 with id = 1 to wxy; + +sql close +sql connect wxy + +sql_error select * from test.st1; +sql insert into test.st1s1 values(now, 10); +sql_error insert into test.st1s2 values(now, 20); +sql_error select * from test.st2; +sql insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); + +system sh/exec.sh -n dnode1 -s stop -x SIGINT From acee9e1862db0f085ddb9feb056c48b9d4a1451d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 23 Apr 2023 08:32:59 +0000 Subject: [PATCH 060/109] fix filter err --- tests/script/tsim/tag/float.sim | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/script/tsim/tag/float.sim b/tests/script/tsim/tag/float.sim index 10fac93d5d..f62feba057 100644 --- a/tests/script/tsim/tag/float.sim +++ b/tests/script/tsim/tag/float.sim @@ -123,6 +123,16 @@ sql select * from $mt where tgcol = 1 if $rows != 100 then return -1 endi + +sql select * from $mt where tgcol = "1.0" +if $rows != 100 then + return -1 +endi + +sql select * from $mt where tgcol = "1" +if $rows != 100 then + return -1 +endi sql select * from $mt where tgcol <> 1 if $rows != 100 then return -1 From af65f9703bf809091c3b6c7579f143c9f80b253f Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sun, 23 Apr 2023 17:36:59 +0800 Subject: [PATCH 061/109] fix:tmq error if consume callback is earlier than consume --- utils/test/c/tmqSim.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/utils/test/c/tmqSim.c b/utils/test/c/tmqSim.c index f2de219f4e..530b142173 100644 --- a/utils/test/c/tmqSim.c +++ b/utils/test/c/tmqSim.c @@ -690,11 +690,12 @@ int32_t notifyMainScript(SThreadInfo* pInfo, int32_t cmdId) { } static int32_t g_once_commit_flag = 0; +static int32_t g_once_consume_flag = 0; static void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { taosFprintfFile(g_fp, "tmq_commit_cb_print() commit %d\n", code); - if (0 == g_once_commit_flag) { + if (g_once_consume_flag == 1 && 0 == g_once_commit_flag) { g_once_commit_flag = 1; notifyMainScript((SThreadInfo*)param, (int32_t)NOTIFY_CMD_START_COMMIT); } @@ -773,8 +774,6 @@ int32_t saveConsumeResult(SThreadInfo* pInfo) { void loop_consume(SThreadInfo* pInfo) { int32_t code; - int32_t once_flag = 0; - int64_t totalMsgs = 0; int64_t totalRows = 0; @@ -834,8 +833,8 @@ void loop_consume(SThreadInfo* pInfo) { lastTotalMsgs = totalMsgs; } - if (0 == once_flag) { - once_flag = 1; + if (0 == g_once_consume_flag) { + g_once_consume_flag = 1; notifyMainScript(pInfo, NOTIFY_CMD_START_CONSUM); } From 623ea6df5fcd8f016dc36bb3303be5bd2e0bb6cd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 23 Apr 2023 17:42:56 +0800 Subject: [PATCH 062/109] fix(stream): add task status check. --- source/libs/stream/src/stream.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 86ba91f76d..046dab380e 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -52,7 +52,7 @@ void streamCleanUp() { void streamSchedByTimer(void* param, void* tmrId) { SStreamTask* pTask = (void*)param; - if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) { + if (streamTaskShouldStop(&pTask->status)) { streamMetaReleaseTask(NULL, pTask); return; } From d23ae400b849f931ccae3db2ff3cd1bdaca00200 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 24 Apr 2023 09:22:59 +0800 Subject: [PATCH 063/109] fix: table meta double free issue --- source/libs/catalog/inc/catalogInt.h | 1 + source/libs/catalog/src/ctgAsync.c | 69 ++++++++++++++++++++++++++++ source/libs/catalog/src/ctgCache.c | 3 +- 3 files changed, 72 insertions(+), 1 deletion(-) diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 1eaf45dafe..c548a6c696 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -359,6 +359,7 @@ typedef struct SCtgSubRes { struct SCtgTask { CTG_TASK_TYPE type; + bool subTask; int32_t taskId; SCtgJob* pJob; void* taskCtx; diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index affcfeb8ac..56c79eac1f 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -699,6 +699,10 @@ _return: } int32_t ctgDumpTbMetaRes(SCtgTask* pTask) { + if (pTask->subTask) { + return TSDB_CODE_SUCCESS; + } + SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pTableMeta) { pJob->jobRes.pTableMeta = taosArrayInit(pJob->tbMetaNum, sizeof(SMetaRes)); @@ -714,6 +718,10 @@ int32_t ctgDumpTbMetaRes(SCtgTask* pTask) { } int32_t ctgDumpTbMetasRes(SCtgTask* pTask) { + if (pTask->subTask) { + return TSDB_CODE_SUCCESS; + } + SCtgJob* pJob = pTask->pJob; pJob->jobRes.pTableMeta = pTask->res; @@ -722,6 +730,10 @@ int32_t ctgDumpTbMetasRes(SCtgTask* pTask) { } int32_t ctgDumpDbVgRes(SCtgTask* pTask) { + if (pTask->subTask) { + return TSDB_CODE_SUCCESS; + } + SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pDbVgroup) { pJob->jobRes.pDbVgroup = taosArrayInit(pJob->dbVgNum, sizeof(SMetaRes)); @@ -737,6 +749,10 @@ int32_t ctgDumpDbVgRes(SCtgTask* pTask) { } int32_t ctgDumpTbHashRes(SCtgTask* pTask) { + if (pTask->subTask) { + return TSDB_CODE_SUCCESS; + } + SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pTableHash) { pJob->jobRes.pTableHash = taosArrayInit(pJob->tbHashNum, sizeof(SMetaRes)); @@ -752,6 +768,10 @@ int32_t ctgDumpTbHashRes(SCtgTask* pTask) { } int32_t ctgDumpTbHashsRes(SCtgTask* pTask) { + if (pTask->subTask) { + return TSDB_CODE_SUCCESS; + } + SCtgJob* pJob = pTask->pJob; pJob->jobRes.pTableHash = pTask->res; @@ -760,6 +780,10 @@ int32_t ctgDumpTbHashsRes(SCtgTask* pTask) { } int32_t ctgDumpTbIndexRes(SCtgTask* pTask) { + if (pTask->subTask) { + return TSDB_CODE_SUCCESS; + } + SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pTableIndex) { SArray* pRes = taosArrayInit(pJob->tbIndexNum, sizeof(SMetaRes)); @@ -779,6 +803,10 @@ int32_t ctgDumpTbIndexRes(SCtgTask* pTask) { } int32_t ctgDumpTbCfgRes(SCtgTask* pTask) { + if (pTask->subTask) { + return TSDB_CODE_SUCCESS; + } + SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pTableCfg) { SArray* pRes = taosArrayInit(pJob->tbCfgNum, sizeof(SMetaRes)); @@ -798,6 +826,10 @@ int32_t ctgDumpTbCfgRes(SCtgTask* pTask) { } int32_t ctgDumpTbTagRes(SCtgTask* pTask) { + if (pTask->subTask) { + return TSDB_CODE_SUCCESS; + } + SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pTableTag) { SArray* pRes = taosArrayInit(pJob->tbTagNum, sizeof(SMetaRes)); @@ -818,6 +850,10 @@ int32_t ctgDumpTbTagRes(SCtgTask* pTask) { int32_t ctgDumpIndexRes(SCtgTask* pTask) { + if (pTask->subTask) { + return TSDB_CODE_SUCCESS; + } + SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pIndex) { pJob->jobRes.pIndex = taosArrayInit(pJob->indexNum, sizeof(SMetaRes)); @@ -833,6 +869,10 @@ int32_t ctgDumpIndexRes(SCtgTask* pTask) { } int32_t ctgDumpQnodeRes(SCtgTask* pTask) { + if (pTask->subTask) { + return TSDB_CODE_SUCCESS; + } + SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pQnodeList) { pJob->jobRes.pQnodeList = taosArrayInit(1, sizeof(SMetaRes)); @@ -848,6 +888,10 @@ int32_t ctgDumpQnodeRes(SCtgTask* pTask) { } int32_t ctgDumpDnodeRes(SCtgTask* pTask) { + if (pTask->subTask) { + return TSDB_CODE_SUCCESS; + } + SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pDnodeList) { pJob->jobRes.pDnodeList = taosArrayInit(1, sizeof(SMetaRes)); @@ -863,6 +907,10 @@ int32_t ctgDumpDnodeRes(SCtgTask* pTask) { } int32_t ctgDumpDbCfgRes(SCtgTask* pTask) { + if (pTask->subTask) { + return TSDB_CODE_SUCCESS; + } + SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pDbCfg) { pJob->jobRes.pDbCfg = taosArrayInit(pJob->dbCfgNum, sizeof(SMetaRes)); @@ -878,6 +926,10 @@ int32_t ctgDumpDbCfgRes(SCtgTask* pTask) { } int32_t ctgDumpDbInfoRes(SCtgTask* pTask) { + if (pTask->subTask) { + return TSDB_CODE_SUCCESS; + } + SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pDbInfo) { pJob->jobRes.pDbInfo = taosArrayInit(pJob->dbInfoNum, sizeof(SMetaRes)); @@ -893,6 +945,10 @@ int32_t ctgDumpDbInfoRes(SCtgTask* pTask) { } int32_t ctgDumpUdfRes(SCtgTask* pTask) { + if (pTask->subTask) { + return TSDB_CODE_SUCCESS; + } + SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pUdfList) { pJob->jobRes.pUdfList = taosArrayInit(pJob->udfNum, sizeof(SMetaRes)); @@ -908,6 +964,10 @@ int32_t ctgDumpUdfRes(SCtgTask* pTask) { } int32_t ctgDumpUserRes(SCtgTask* pTask) { + if (pTask->subTask) { + return TSDB_CODE_SUCCESS; + } + SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pUser) { pJob->jobRes.pUser = taosArrayInit(pJob->userNum, sizeof(SMetaRes)); @@ -923,6 +983,10 @@ int32_t ctgDumpUserRes(SCtgTask* pTask) { } int32_t ctgDumpSvrVer(SCtgTask* pTask) { + if (pTask->subTask) { + return TSDB_CODE_SUCCESS; + } + SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pSvrVer) { pJob->jobRes.pSvrVer = taosMemoryCalloc(1, sizeof(SMetaRes)); @@ -2239,6 +2303,8 @@ int32_t ctgLaunchGetUserTask(SCtgTask* pTask) { if (inCache) { pTask->res = rsp.pRawRes; + ctgTaskDebug("Final res got, pass:%d, pCond:%p", rsp.pRawRes->pass, rsp.pRawRes->pCond); + CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); return TSDB_CODE_SUCCESS; } @@ -2475,6 +2541,9 @@ int32_t ctgLaunchSubTask(SCtgTask* pTask, CTG_TASK_TYPE type, ctgSubTaskCbFp fp, } SCtgTask* pSub = taosArrayGet(pJob->pTasks, subTaskId); + if (newTask) { + pSub->subTask = true; + } CTG_ERR_RET(ctgSetSubTaskCb(pSub, pTask)); diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index ee864d985e..0be2a78d48 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -765,8 +765,9 @@ _return: *inCache = false; CTG_CACHE_NHIT_INC(CTG_CI_USER, 1); + ctgDebug("Get user from cache failed, user:%s, metaNotExists:%d, code:%d", pReq->user, pRes->metaNotExists, code); - return TSDB_CODE_SUCCESS; + return code; } void ctgDequeue(SCtgCacheOperation **op) { From 8f17c2f2d7225ac8c4a3d52d16bb44c306ace99e Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 24 Apr 2023 09:37:51 +0800 Subject: [PATCH 064/109] test:modify base version in compatibility.py --- tests/system-test/0-others/compatibility.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index 33d1dac4b5..22e319fdaf 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -17,12 +17,12 @@ from util.dnodes import TDDnode from util.cluster import * import subprocess -BASEVERSION = "3.0.1.8" +BASEVERSION = "3.0.2.3" class TDTestCase: def caseDescription(self): - ''' + f''' 3.0 data compatibility test - case1: basedata version is 3.0.1.8 + case1: basedata version is {BASEVERSION} ''' return From 7e9432606d245dc4df81c06e4d78135a627c1b2b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 24 Apr 2023 09:50:39 +0800 Subject: [PATCH 065/109] fix(tmq): revoke the fix, it will definitly cause the deadlock. --- tests/system-test/7-tmq/tmqCommon.py | 2 +- utils/test/c/tmqSim.c | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index f63c70a4c6..32bb22c8cc 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -170,7 +170,7 @@ class TMQCom: if tdSql.getData(i, 1) == 1: loopFlag = 0 break - time.sleep(0.02) + time.sleep(0.10) return def create_database(self,tsql, dbName,dropFlag=1,vgroups=4,replica=1): diff --git a/utils/test/c/tmqSim.c b/utils/test/c/tmqSim.c index 530b142173..f2de219f4e 100644 --- a/utils/test/c/tmqSim.c +++ b/utils/test/c/tmqSim.c @@ -690,12 +690,11 @@ int32_t notifyMainScript(SThreadInfo* pInfo, int32_t cmdId) { } static int32_t g_once_commit_flag = 0; -static int32_t g_once_consume_flag = 0; static void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { taosFprintfFile(g_fp, "tmq_commit_cb_print() commit %d\n", code); - if (g_once_consume_flag == 1 && 0 == g_once_commit_flag) { + if (0 == g_once_commit_flag) { g_once_commit_flag = 1; notifyMainScript((SThreadInfo*)param, (int32_t)NOTIFY_CMD_START_COMMIT); } @@ -774,6 +773,8 @@ int32_t saveConsumeResult(SThreadInfo* pInfo) { void loop_consume(SThreadInfo* pInfo) { int32_t code; + int32_t once_flag = 0; + int64_t totalMsgs = 0; int64_t totalRows = 0; @@ -833,8 +834,8 @@ void loop_consume(SThreadInfo* pInfo) { lastTotalMsgs = totalMsgs; } - if (0 == g_once_consume_flag) { - g_once_consume_flag = 1; + if (0 == once_flag) { + once_flag = 1; notifyMainScript(pInfo, NOTIFY_CMD_START_CONSUM); } From 4b166e567e1ee6eb77c1bdeffa494b59e32db563 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 24 Apr 2023 10:14:02 +0800 Subject: [PATCH 066/109] fix: fix client retry issue --- source/client/src/clientMain.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 60c7b44b3d..54ae1ab4b3 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -975,8 +975,10 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { if (TSDB_CODE_SUCCESS == code) { pRequest->stmtType = pRequest->pQuery->pRoot->type; - phaseAsyncQuery(pWrapper); - } else { + code = phaseAsyncQuery(pWrapper); + } + + if (TSDB_CODE_SUCCESS != code) { tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code), pRequest->requestId); destorySqlCallbackWrapper(pWrapper); From af5bcb80bffddce7687a2b41efc220851ff9cb1c Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 24 Apr 2023 10:26:40 +0800 Subject: [PATCH 067/109] opti:test cases for tmq --- .../6-cluster/clusterCommonCreate.py | 40 ++++++++--------- tests/system-test/7-tmq/subscribeDb3.py | 45 ++++++++++--------- tests/system-test/7-tmq/tmqCommon.py | 22 +++++---- tests/system-test/7-tmq/tmqConsumerGroup.py | 2 +- tests/system-test/7-tmq/tmqDnodeRestart1.py | 4 +- 5 files changed, 56 insertions(+), 57 deletions(-) diff --git a/tests/system-test/6-cluster/clusterCommonCreate.py b/tests/system-test/6-cluster/clusterCommonCreate.py index 6e699e2396..a06c1233d8 100644 --- a/tests/system-test/6-cluster/clusterCommonCreate.py +++ b/tests/system-test/6-cluster/clusterCommonCreate.py @@ -94,26 +94,26 @@ class ClusterComCreate: tdLog.info(shellCmd) os.system(shellCmd) - def getStartConsumeNotifyFromTmqsim(self,cdbName='cdb'): - while 1: - tdSql.query("select * from %s.notifyinfo"%cdbName) - #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) - if (tdSql.getRows() == 1) and (tdSql.getData(0, 1) == 0): - break - else: - time.sleep(0.1) - return - - def getStartCommitNotifyFromTmqsim(self,cdbName='cdb'): - while 1: - tdSql.query("select * from %s.notifyinfo"%cdbName) - #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) - if tdSql.getRows() == 2 : - print(tdSql.getData(0, 1), tdSql.getData(1, 1)) - if tdSql.getData(1, 1) == 1: - break - time.sleep(0.1) - return + # def getStartConsumeNotifyFromTmqsim(self,cdbName='cdb'): + # while 1: + # tdSql.query("select * from %s.notifyinfo"%cdbName) + # #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + # if (tdSql.getRows() == 1) and (tdSql.getData(0, 1) == 0): + # break + # else: + # time.sleep(0.1) + # return + # + # def getStartCommitNotifyFromTmqsim(self,cdbName='cdb'): + # while 1: + # tdSql.query("select * from %s.notifyinfo"%cdbName) + # #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + # if tdSql.getRows() == 2 : + # print(tdSql.getData(0, 1), tdSql.getData(1, 1)) + # if tdSql.getData(1, 1) == 1: + # break + # time.sleep(0.1) + # return def create_database(self,tsql, dbName,dropFlag=1,vgroups=4,replica=1): if dropFlag == 1: diff --git a/tests/system-test/7-tmq/subscribeDb3.py b/tests/system-test/7-tmq/subscribeDb3.py index 1de9b62bcd..22004a95f1 100644 --- a/tests/system-test/7-tmq/subscribeDb3.py +++ b/tests/system-test/7-tmq/subscribeDb3.py @@ -10,6 +10,7 @@ from util.log import * from util.sql import * from util.cases import * from util.dnodes import * +from tmqCommon import * class TDTestCase: hostname = socket.gethostname() @@ -67,26 +68,26 @@ class TDTestCase: tdLog.info("consume info sql: %s"%sql) tdSql.query(sql) - def getStartConsumeNotifyFromTmqsim(self,cdbName='cdb'): - while 1: - tdSql.query("select * from %s.notifyinfo"%cdbName) - #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) - if (tdSql.getRows() == 1) and (tdSql.getData(0, 1) == 0): - break - else: - time.sleep(0.1) - return - - def getStartCommitNotifyFromTmqsim(self,cdbName='cdb'): - while 1: - tdSql.query("select * from %s.notifyinfo"%cdbName) - #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) - if tdSql.getRows() == 2 : - tdLog.info("row[0][1]: %d, row[1][1]: %d"%(tdSql.getData(0, 1), tdSql.getData(1, 1))) - if tdSql.getData(1, 1) == 1: - break - time.sleep(0.1) - return + # def getStartConsumeNotifyFromTmqsim(self,cdbName='cdb'): + # while 1: + # tdSql.query("select * from %s.notifyinfo"%cdbName) + # #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + # if (tdSql.getRows() == 1) and (tdSql.getData(0, 1) == 0): + # break + # else: + # time.sleep(0.1) + # return + # + # def getStartCommitNotifyFromTmqsim(self,cdbName='cdb'): + # while 1: + # tdSql.query("select * from %s.notifyinfo"%cdbName) + # #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + # if tdSql.getRows() == 2 : + # tdLog.info("row[0][1]: %d, row[1][1]: %d"%(tdSql.getData(0, 1), tdSql.getData(1, 1))) + # if tdSql.getData(1, 1) == 1: + # break + # time.sleep(0.1) + # return def selectConsumeResult(self,expectRows,cdbName='cdb'): resultList=[] @@ -233,7 +234,7 @@ class TDTestCase: self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) tdLog.info("wait the notify info of start consume") - self.getStartConsumeNotifyFromTmqsim() + tmqCom.getStartConsumeNotifyFromTmqsim() tdLog.info("pkill consume processor") if (platform.system().lower() == 'windows'): @@ -311,7 +312,7 @@ class TDTestCase: # time.sleep(6) tdLog.info("start to wait commit notify") - self.getStartCommitNotifyFromTmqsim() + tmqCom.getStartCommitNotifyFromTmqsim() tdLog.info("pkill consume processor") if (platform.system().lower() == 'windows'): diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index f63c70a4c6..44eef8bf24 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -145,31 +145,29 @@ class TMQCom: processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") tdLog.debug("%s is stopped by kill -INT" % (processorName)) - def getStartConsumeNotifyFromTmqsim(self,cdbName='cdb',rows=1): + def getStartConsumeNotifyFromTmqsim(self,cdbName='cdb'): loopFlag = 1 while loopFlag: tdSql.query("select * from %s.notifyinfo"%cdbName) #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) actRows = tdSql.getRows() - if (actRows >= rows): - for i in range(actRows): - if tdSql.getData(i, 1) == 0: - loopFlag = 0 - break + for i in range(actRows): + if tdSql.getData(i, 1) == 0: + loopFlag = 0 + break time.sleep(0.02) return - def getStartCommitNotifyFromTmqsim(self,cdbName='cdb',rows=2): + def getStartCommitNotifyFromTmqsim(self,cdbName='cdb'): loopFlag = 1 while loopFlag: tdSql.query("select * from %s.notifyinfo"%cdbName) #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) actRows = tdSql.getRows() - if (actRows >= rows): - for i in range(actRows): - if tdSql.getData(i, 1) == 1: - loopFlag = 0 - break + for i in range(actRows): + if tdSql.getData(i, 1) == 1: + loopFlag = 0 + break time.sleep(0.02) return diff --git a/tests/system-test/7-tmq/tmqConsumerGroup.py b/tests/system-test/7-tmq/tmqConsumerGroup.py index f05f600f27..ae9671bcf4 100644 --- a/tests/system-test/7-tmq/tmqConsumerGroup.py +++ b/tests/system-test/7-tmq/tmqConsumerGroup.py @@ -100,7 +100,7 @@ class TDTestCase: tdLog.info("wait consumer commit notify") # tmqCom.getStartCommitNotifyFromTmqsim(rows=4) - tmqCom.getStartConsumeNotifyFromTmqsim(rows=2) + tmqCom.getStartConsumeNotifyFromTmqsim() tdLog.info("pkill one consume processor") tmqCom.stopTmqSimProcess('tmq_sim_new') diff --git a/tests/system-test/7-tmq/tmqDnodeRestart1.py b/tests/system-test/7-tmq/tmqDnodeRestart1.py index cff55a1239..2bde32800b 100644 --- a/tests/system-test/7-tmq/tmqDnodeRestart1.py +++ b/tests/system-test/7-tmq/tmqDnodeRestart1.py @@ -121,7 +121,7 @@ class TDTestCase: tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) # time.sleep(3) - tmqCom.getStartCommitNotifyFromTmqsim('cdb',1) + tmqCom.getStartCommitNotifyFromTmqsim() tdLog.info("create some new child table and insert data for latest mode") paraDict["batchNum"] = 100 @@ -205,7 +205,7 @@ class TDTestCase: tdLog.info("start consume processor") tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) - tmqCom.getStartCommitNotifyFromTmqsim('cdb',1) + tmqCom.getStartCommitNotifyFromTmqsim() tdLog.info("create some new child table and insert data for latest mode") paraDict["batchNum"] = 10 From 76810b81660803112a4e6b20e42cb65fe3910e39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Mon, 24 Apr 2023 10:29:35 +0800 Subject: [PATCH 068/109] test: refine query cases --- .../system-test/1-insert/database_pre_suf.py | 264 +++++++++++++++--- 1 file changed, 229 insertions(+), 35 deletions(-) diff --git a/tests/system-test/1-insert/database_pre_suf.py b/tests/system-test/1-insert/database_pre_suf.py index a6ff95ab3f..2e993b9a40 100755 --- a/tests/system-test/1-insert/database_pre_suf.py +++ b/tests/system-test/1-insert/database_pre_suf.py @@ -24,9 +24,7 @@ from util.dnodes import tdDnodes from util.dnodes import * class TDTestCase: - updatecfgDict = {'maxSQLLength':1048576,'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , - "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, - "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + updatecfgDict = {'maxSQLLength':1048576,'debugFlag': 135} def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) @@ -39,7 +37,9 @@ class TDTestCase: self.db = "pre_suf" - def dropandcreateDB_random(self,database,n,vgroups,table_prefix,table_suffix,check_result): + def dropandcreateDB_random(self,database,n,vgroups,table_prefix,table_suffix,check_result_positive,check_result_negative): + #check_result_positive 检查前缀后缀是正数的，check_result_negative 检查前缀后缀是负数的(TS-3249) + tdLog.info(f"create start:n:{n},vgroups:{vgroups},table_prefix:{table_prefix},table_suffix:{table_suffix},check_result_positive:{check_result_positive},check_result_negative:{check_result_negative}") ts = 1630000000000 num_random = 100 fake = Faker('zh_CN') @@ -56,6 +56,7 @@ class TDTestCase: q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) \ tags(loc nchar(100) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, t_bool bool , t_binary binary(100) , t_nchar nchar(100) ,t_float float , t_double double , t_ts timestamp);''') + #positive for i in range(10*n): tdSql.execute('''create table bj_%d (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp ) ;'''%i) tdSql.execute('''create table sh_%d (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp ) ;'''%i) @@ -106,11 +107,60 @@ class TDTestCase: fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , fake.pystr() ,fake.pystr() ,fake.pyfloat(),fake.pyfloat(),fake.random_int(min=-2147483647, max=2147483647, step=1))) - + #negative + for i in range(10*n): + tdSql.execute('''create table bj_table_%d_r_negative (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp ) ;'''%i) + tdSql.execute('''create table sh_table_%d_r_negative (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp ) ;'''%i) + tdSql.execute('''create table hn_table_%d_r_negative \ + (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , \ + q_binary1 binary(100) , q_nchar1 nchar(100) ,q_binary2 binary(100) , q_nchar2 nchar(100) ,q_binary3 binary(100) , q_nchar3 nchar(100) ,q_binary4 binary(100) , q_nchar4 nchar(100) ,\ + q_binary5 binary(100) , q_nchar5 nchar(100) ,q_binary6 binary(100) , q_nchar6 nchar(100) ,q_binary7 binary(100) , q_nchar7 nchar(100) ,q_binary8 binary(100) , q_nchar8 nchar(100) ,\ + q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) ;'''%i) + + tdSql.execute('''create table bj_stable_1_%d_negative using stable_1 tags('bj_stable_1_%d', '%d' , '%d', '%d' , '%d' , 1 , 'binary1.%s' , 'nchar1.%s' , '%f', '%f' ,'%d') ;''' + %(i,i,fake.random_int(min=-2147483647, max=2147483647, step=1), fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1), + fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , + fake.pystr() ,fake.pystr() ,fake.pyfloat(),fake.pyfloat(),fake.random_int(min=-2147483647, max=2147483647, step=1))) + tdSql.execute('''create table sh_table_%d_a_negative using stable_1 tags('sh_a_table_1_%d', '%d' , '%d', '%d' , '%d' , 1 , 'binary1.%s' , 'nchar1.%s' , '%f', '%f' ,'%d') ;''' + %(i,i,fake.random_int(min=-2147483647, max=2147483647, step=1), fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1), + fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , + fake.pystr() ,fake.pystr() ,fake.pyfloat(),fake.pyfloat(),fake.random_int(min=-2147483647, max=2147483647, step=1))) + tdSql.execute('''create table sh_table_%d_b_negative using stable_1 tags('sh_b_table_1_%d', '%d' , '%d', '%d' , '%d' , 1 , 'binary1.%s' , 'nchar1.%s' , '%f', '%f' ,'%d') ;''' + %(i,i,fake.random_int(min=-2147483647, max=2147483647, step=1), fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1), + fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , + fake.pystr() ,fake.pystr() ,fake.pyfloat(),fake.pyfloat(),fake.random_int(min=-2147483647, max=2147483647, step=1))) + tdSql.execute('''create table sh_table_%d_c_negative using stable_1 tags('sh_c_table_1_%d', '%d' , '%d', '%d' , '%d' , 1 , 'binary1.%s' , 'nchar1.%s' , '%f', '%f' ,'%d') ;''' + %(i,i,fake.random_int(min=-2147483647, max=2147483647, step=1), fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1), + fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , + fake.pystr() ,fake.pystr() ,fake.pyfloat(),fake.pyfloat(),fake.random_int(min=-2147483647, max=2147483647, step=1))) + + tdSql.execute('''create table bj_table_%d_a_negative using stable_1 tags('bj_a_table_1_%d', '%d' , '%d', '%d' , '%d' , 1 , 'binary1.%s' , 'nchar1.%s' , '%f', '%f' ,'%d') ;''' + %(i,i,fake.random_int(min=-2147483647, max=2147483647, step=1), fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1), + fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , + fake.pystr() ,fake.pystr() ,fake.pyfloat(),fake.pyfloat(),fake.random_int(min=-2147483647, max=2147483647, step=1))) + tdSql.execute('''create table bj_table_%d_b_negative using stable_1 tags('bj_b_table_1_%d', '%d' , '%d', '%d' , '%d' , 1 , 'binary1.%s' , 'nchar1.%s' , '%f', '%f' ,'%d') ;''' + %(i,i,fake.random_int(min=-2147483647, max=2147483647, step=1), fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1), + fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , + fake.pystr() ,fake.pystr() ,fake.pyfloat(),fake.pyfloat(),fake.random_int(min=-2147483647, max=2147483647, step=1))) + tdSql.execute('''create table bj_table_%d_c_negative using stable_1 tags('bj_c_table_1_%d', '%d' , '%d', '%d' , '%d' , 1 , 'binary1.%s' , 'nchar1.%s' , '%f', '%f' ,'%d') ;''' + %(i,i,fake.random_int(min=-2147483647, max=2147483647, step=1), fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1), + fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , + fake.pystr() ,fake.pystr() ,fake.pyfloat(),fake.pyfloat(),fake.random_int(min=-2147483647, max=2147483647, step=1))) + + + tdSql.execute('''create table tj_table_%d_a_negative using stable_2 tags('tj_a_table_2_%d', '%d' , '%d', '%d' , '%d' , 1 , 'binary1.%s' , 'nchar1.%s' , '%f', '%f' ,'%d') ;''' + %(i,i,fake.random_int(min=-2147483647, max=2147483647, step=1), fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1), + fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , + fake.pystr() ,fake.pystr() ,fake.pyfloat(),fake.pyfloat(),fake.random_int(min=-2147483647, max=2147483647, step=1))) + tdSql.execute('''create table tj_table_%d_b_negative using stable_2 tags('tj_b_table_2_%d', '%d' , '%d', '%d' , '%d' , 1 , 'binary1.%s' , 'nchar1.%s' , '%f', '%f' ,'%d') ;''' + %(i,i,fake.random_int(min=-2147483647, max=2147483647, step=1), fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1), + fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , + fake.pystr() ,fake.pystr() ,fake.pyfloat(),fake.pyfloat(),fake.random_int(min=-2147483647, max=2147483647, step=1))) + # create stream tdSql.execute('''create stream current_stream trigger at_once IGNORE EXPIRED 0 into stream_max_stable_1 as select _wstart as startts, _wend as wend, max(q_int) as max_int, min(q_bigint) as min_int from stable_1 where ts is not null interval (5s);''') - # insert data + # insert data positive for i in range(num_random*n): tdSql.execute('''insert into bj_stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts,\ q_binary1 , q_nchar1 , q_binary2 , q_nchar2 , q_binary3 , q_nchar3 , q_binary4 , q_nchar4 , q_binary5 , q_nchar5 , q_binary6 , q_nchar6 , q_binary7 , q_nchar7, q_binary8 , q_nchar8) \ @@ -180,8 +230,60 @@ class TDTestCase: fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i, fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr())) + # insert data negative + for i in range(num_random*n): + tdSql.execute('''insert into bj_stable_1_1_negative (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts,\ + q_binary1 , q_nchar1 , q_binary2 , q_nchar2 , q_binary3 , q_nchar3 , q_binary4 , q_nchar4 , q_binary5 , q_nchar5 , q_binary6 , q_nchar6 , q_binary7 , q_nchar7, q_binary8 , q_nchar8) \ + values(%d, %d, %d, %d, %d, %f, %f, 0, 'binary.%s', 'nchar.%s', %d, 'binary1.%s', 'nchar1.%s', 'binary2.%s', 'nchar2.%s', 'binary3.%s', 'nchar3.%s', \ + 'binary4.%s', 'nchar4.%s', 'binary5.%s', 'nchar5.%s', 'binary6.%s', 'nchar6.%s', 'binary7.%s', 'nchar7.%s', 'binary8.%s', 'nchar8.%s') ;''' + % (ts + i*1000, fake.random_int(min=-2147483647, max=2147483647, step=1), + fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1), + fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , + fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , + fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr())) + + tdSql.execute('''insert into bj_stable_1_2_negative (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts,\ + q_binary1 , q_nchar1 , q_binary2 , q_nchar2 , q_binary3 , q_nchar3 , q_binary4 , q_nchar4 , q_binary5 , q_nchar5 , q_binary6 , q_nchar6 , q_binary7 , q_nchar7, q_binary8 , q_nchar8)\ + values(%d, %d, %d, %d, %d, %f, %f, 1, 'binary.%s', 'nchar.%s', %d, 'binary1.%s', 'nchar1.%s', 'binary2.%s', 'nchar2.%s', 'binary3.%s', 'nchar3.%s', \ + 'binary4.%s', 'nchar4.%s', 'binary5.%s', 'nchar5.%s', 'binary6.%s', 'nchar6.%s', 'binary7.%s', 'nchar7.%s', 'binary8.%s', 'nchar8.%s') ;''' + % (ts + i*1000, fake.random_int(min=0, max=2147483647, step=1), + fake.random_int(min=0, max=9223372036854775807, step=1), + fake.random_int(min=0, max=32767, step=1) , fake.random_int(min=0, max=127, step=1) , + fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i, fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , + fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr())) + + tdSql.execute('''insert into bj_stable_1_3_negative (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts,\ + q_binary1 , q_nchar1 , q_binary2 , q_nchar2 , q_binary3 , q_nchar3 , q_binary4 , q_nchar4 , q_binary5 , q_nchar5 , q_binary6 , q_nchar6 , q_binary7 , q_nchar7, q_binary8 , q_nchar8) \ + values(%d, %d, %d, %d, %d, %f, %f, 0, 'binary.%s', 'nchar.%s', %d, 'binary1.%s', 'nchar1.%s', 'binary2.%s', 'nchar2.%s', 'binary3.%s', 'nchar3.%s', \ + 'binary4.%s', 'nchar4.%s', 'binary5.%s', 'nchar5.%s', 'binary6.%s', 'nchar6.%s', 'binary7.%s', 'nchar7.%s', 'binary8.%s', 'nchar8.%s') ;''' + % (ts + i*1000, fake.random_int(min=-0, max=2147483647, step=1), + fake.random_int(min=-0, max=9223372036854775807, step=1), + fake.random_int(min=-0, max=32767, step=1) , fake.random_int(min=-0, max=127, step=1) , + fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i, fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , + fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr())) + + tdSql.execute('''insert into bj_stable_1_4_negative (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts,\ + q_binary1 , q_nchar1 , q_binary2 , q_nchar2 , q_binary3 , q_nchar3 , q_binary4 , q_nchar4 , q_binary5 , q_nchar5 , q_binary6 , q_nchar6 , q_binary7 , q_nchar7, q_binary8 , q_nchar8) \ + values(%d, %d, %d, %d, %d, %f, %f, 0, 'binary.%s', 'nchar.%s', %d, 'binary1.%s', 'nchar1.%s', 'binary2.%s', 'nchar2.%s', 'binary3.%s', 'nchar3.%s', \ + 'binary4.%s', 'nchar4.%s', 'binary5.%s', 'nchar5.%s', 'binary6.%s', 'nchar6.%s', 'binary7.%s', 'nchar7.%s', 'binary8.%s', 'nchar8.%s') ;''' + % (ts + i*1000 +1, fake.random_int(min=-0, max=2147483647, step=1), + fake.random_int(min=-0, max=9223372036854775807, step=1), + fake.random_int(min=-0, max=32767, step=1) , fake.random_int(min=-0, max=127, step=1) , + fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i, fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , + fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr())) + + tdSql.execute('''insert into bj_stable_1_5_negative (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts,\ + q_binary1 , q_nchar1 , q_binary2 , q_nchar2 , q_binary3 , q_nchar3 , q_binary4 , q_nchar4 , q_binary5 , q_nchar5 , q_binary6 , q_nchar6 , q_binary7 , q_nchar7, q_binary8 , q_nchar8) \ + values(%d, %d, %d, %d, %d, %f, %f, 0, 'binary.%s', 'nchar.%s', %d, 'binary1.%s', 'nchar1.%s', 'binary2.%s', 'nchar2.%s', 'binary3.%s', 'nchar3.%s', \ + 'binary4.%s', 'nchar4.%s', 'binary5.%s', 'nchar5.%s', 'binary6.%s', 'nchar6.%s', 'binary7.%s', 'nchar7.%s', 'binary8.%s', 'nchar8.%s') ;''' + % (ts + i*1000 +10, fake.random_int(min=-0, max=2147483647, step=1), + fake.random_int(min=-0, max=9223372036854775807, step=1), + fake.random_int(min=-0, max=32767, step=1) , fake.random_int(min=-0, max=127, step=1) , + fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i, fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , + fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr() , fake.pystr())) + tdSql.query("select count(*) from stable_1;") - tdSql.checkData(0,0,5*num_random*n) + tdSql.checkData(0,0,10*num_random*n) tdSql.query("select count(*) from hn_table_1_r;") tdSql.checkData(0,0,num_random*n) @@ -220,39 +322,28 @@ class TDTestCase: tdSql.query(" select * from information_schema.ins_databases where name = '%s';" %database) - print(tdSql.queryResult) + tdLog.info(tdSql.queryResult) - # tdSql.query(" select table_prefix,table_suffix from information_schema.ins_databases where name = '%s';" %database) - # print(tdSql.queryResult) - #TD-19082 - - #tdSql.query(" select * from information_schema.ins_tables where db_name = '%s';" %database) - #print(tdSql.queryResult) - - tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s';" %database) + tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' limit 3;" %database) queryRows = len(tdSql.queryResult) for i in range(queryRows): - print("row=%d, vgroup_id=%s, tbname=%s " %(i,tdSql.queryResult[i][1],tdSql.queryResult[i][0])) + tdLog.info("row=%d, vgroup_id=%s, tbname=%s " %(i,tdSql.queryResult[i][1],tdSql.queryResult[i][0])) tdLog.info("\n=============flush database ====================\n") tdSql.execute(" flush database %s;" %database) tdSql.query(" select * from information_schema.ins_databases where name = '%s';" %database) - print(tdSql.queryResult) - - # tdSql.query(" select table_prefix,table_suffix from information_schema.ins_databases where name = '%s';" %database) - # print(tdSql.queryResult) - #TD-19082 - - tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s';" %database) + tdLog.info(tdSql.queryResult) + + tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' limit 3;" %database) queryRows = len(tdSql.queryResult) for i in range(queryRows): - print("row=%d, vgroup_id=%s, tbname=%s " %(i,tdSql.queryResult[i][1],tdSql.queryResult[i][0])) + tdLog.info("row=%d, vgroup_id=%s, tbname=%s " %(i,tdSql.queryResult[i][1],tdSql.queryResult[i][0])) # check in one vgroup - if check_result == 'Y': + if check_result_positive == 'Y': #base table : sh_table_0_a tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name='sh_table_0_a';" %(database)) base_value_table_name = tdSql.queryResult[0][0] @@ -324,8 +415,100 @@ class TDTestCase: tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name='tj_table_%d_b';" %(database,i)) self.value_check(base_value_table_name,base_value_table_vgroup) + elif check_result_negative == 'Y': + #base table : sh_table_0_a + tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name='sh_table_0_a_negative';" %(database)) + base_value_table_name = tdSql.queryResult[0][0] + base_value_table_vgroup = tdSql.queryResult[0][1] + + #check table :sh_table_i_a + check_rows = tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name like 'sh_table_%%_a_negative';" %(database)) + for i in range(check_rows): + tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name='sh_table_%d_a_negative';" %(database,i)) + self.value_check(base_value_table_name,base_value_table_vgroup) + + #check table :sh_table_i_b + check_rows = tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name like 'sh_table_%%_b_negative';" %(database)) + for i in range(check_rows): + tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name='sh_table_%d_b_negative';" %(database,i)) + self.value_check(base_value_table_name,base_value_table_vgroup) + + #check table :sh_table_i_c + check_rows = tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name like 'sh_table_%%_c_negative';" %(database)) + for i in range(check_rows): + tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name='sh_table_%d_c_negative';" %(database,i)) + self.value_check(base_value_table_name,base_value_table_vgroup) + + #check table :sh_table_i_r + check_rows = tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name like 'sh_table_%%_r_negative';" %(database)) + for i in range(check_rows): + tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name='sh_table_%d_r_negative';" %(database,i)) + self.value_check(base_value_table_name,base_value_table_vgroup) + + #base table : bj_table_0_a + tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name='bj_table_0_a_negative';" %(database)) + base_value_table_name = tdSql.queryResult[0][0] + base_value_table_vgroup = tdSql.queryResult[0][1] + + #check table :bj_table_i_a + check_rows = tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name like 'bj_table_%%_a_negative';" %(database)) + for i in range(check_rows): + tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name='bj_table_%d_a_negative';" %(database,i)) + self.value_check(base_value_table_name,base_value_table_vgroup) + + #check table :bj_table_i_b + check_rows = tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name like 'bj_table_%%_b_negative';" %(database)) + for i in range(check_rows): + tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name='bj_table_%d_b_negative';" %(database,i)) + self.value_check(base_value_table_name,base_value_table_vgroup) + + #check table :bj_table_i_c + check_rows = tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name like 'bj_table_%%_c_negative';" %(database)) + for i in range(check_rows): + tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name='bj_table_%d_c_negative';" %(database,i)) + self.value_check(base_value_table_name,base_value_table_vgroup) + + #check table :bj_table_i_r + check_rows = tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name like 'bj_table_%%_r_negative';" %(database)) + for i in range(check_rows): + tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name='bj_table_%d_r_negative';" %(database,i)) + self.value_check(base_value_table_name,base_value_table_vgroup) + + #base table : hn_table_0_r + tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name='hn_table_0_r_negative';" %(database)) + base_value_table_name = tdSql.queryResult[0][0] + base_value_table_vgroup = tdSql.queryResult[0][1] + + #check table :hn_table_i_r + check_rows = tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name like 'hn_table_%%_r_negative';" %(database)) + for i in range(check_rows): + tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name='hn_table_%d_r_negative';" %(database,i)) + self.value_check(base_value_table_name,base_value_table_vgroup) + + + #base table : tj_table_0_r + tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name='tj_table_0_a_negative';" %(database)) + base_value_table_name = tdSql.queryResult[0][0] + base_value_table_vgroup = tdSql.queryResult[0][1] + + #check table :tj_table_i_a + check_rows = tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name like 'tj_table_%%_a_negative';" %(database)) + for i in range(check_rows): + tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name='tj_table_%d_a_negative';" %(database,i)) + self.value_check(base_value_table_name,base_value_table_vgroup) + + #check table :tj_table_i_b + check_rows = tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name like 'tj_table_%%_b_negative';" %(database)) + for i in range(check_rows): + tdSql.query(" select table_name,vgroup_id from information_schema.ins_tables where db_name = '%s' and table_name='tj_table_%d_b_negative';" %(database,i)) + self.value_check(base_value_table_name,base_value_table_vgroup) + + else: pass + + tdLog.info(f"create end:n:{n},vgroups:{vgroups},table_prefix:{table_prefix},table_suffix:{table_suffix},check_result_positive:{check_result_positive},check_result_negative:{check_result_negative}") + def value_check(self,base_value_table_name,base_value_table_vgroup): check_value_table_name = tdSql.queryResult[0][0] @@ -348,17 +531,28 @@ class TDTestCase: os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename)) - #(self,database,n,vgroups,table_prefix,table_suffix) - self.dropandcreateDB_random("%s" %self.db, 1,2,0,0,'N') - self.dropandcreateDB_random("%s" %self.db, 1,2,0,2,'N') - self.dropandcreateDB_random("%s" %self.db, 1,2,2,0,'N') - self.dropandcreateDB_random("%s" %self.db, 1,2,3,3,'Y') - self.dropandcreateDB_random("%s" %self.db, 1,3,3,3,'Y') - self.dropandcreateDB_random("%s" %self.db, 1,4,4,4,'Y') - self.dropandcreateDB_random("%s" %self.db, 1,5,5,5,'Y') + #(self,database,n,vgroups,table_prefix,table_suffix,check_result_positive,check_result_negative): + #check_result_positive 检查前缀后缀是正数的，check_result_negative 检查前缀后缀是负数的(TS-3249) + # self.dropandcreateDB_random("%s" %self.db, 1,2,0,0,'N','N') + # self.dropandcreateDB_random("%s" %self.db, 1,2,0,2,'N','N') + # self.dropandcreateDB_random("%s" %self.db, 1,2,2,0,'N','N') + + self.dropandcreateDB_random("%s" %self.db, 1,random.randint(1,5),random.randint(0,3),random.randint(0,3),'N','N') + self.dropandcreateDB_random("%s" %self.db, 1,random.randint(1,5),random.randint(-10,0),random.randint(-10,0),'N','N') + self.dropandcreateDB_random("%s" %self.db, 1,random.randint(1,5),random.randint(-191,0),random.randint(-191,0),'N','N') + self.dropandcreateDB_random("%s" %self.db, 1,random.randint(1,5),random.randint(0,100),random.randint(0,91),'N','N') + + # self.dropandcreateDB_random("%s" %self.db, 1,2,3,3,'Y','N') + # self.dropandcreateDB_random("%s" %self.db, 1,3,3,3,'Y','N') + # self.dropandcreateDB_random("%s" %self.db, 1,4,4,4,'Y','N') + # self.dropandcreateDB_random("%s" %self.db, 1,5,5,5,'Y','N') + self.dropandcreateDB_random("%s" %self.db, 1,random.randint(1,5),random.randint(3,5),random.randint(3,5),'Y','N') + + self.dropandcreateDB_random("%s" %self.db, 1,random.randint(1,5),random.randint(-5,-1),0,'N','Y') + self.dropandcreateDB_random("%s" %self.db, 1,random.randint(1,5),random.randint(-5,-1),random.randint(-9,-0),'N','Y') - #taos -f sql + # #taos -f sql print("taos -f sql start!") taos_cmd1 = "taos -f %s/%s.sql" % (self.testcasePath,self.testcaseFilename) _ = subprocess.check_output(taos_cmd1, shell=True) From 79605289932b6062f8ae8004b41331b38642e93e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Mon, 24 Apr 2023 10:29:47 +0800 Subject: [PATCH 069/109] test: refine query cases --- tests/system-test/2-query/slimit.py | 1821 +++++++++++++++++++++++++++ 1 file changed, 1821 insertions(+) create mode 100644 tests/system-test/2-query/slimit.py diff --git a/tests/system-test/2-query/slimit.py b/tests/system-test/2-query/slimit.py new file mode 100644 index 0000000000..48209da59a --- /dev/null +++ b/tests/system-test/2-query/slimit.py @@ -0,0 +1,1821 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +import random +import re + +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql, replicaVar): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.testcasePath = os.path.split(__file__)[0] + self.testcaseFilename = os.path.split(__file__)[-1] + os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename)) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run_benchmark(self,dbname,tables,per_table_num,vgroups,replica): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + + print("%staosBenchmark -d %s -t %d -n %d -v %d -a %d -y " % (binPath,dbname,tables,per_table_num,vgroups,replica)) + os.system("%staosBenchmark -d %s -t %d -n %d -v %d -a %d -y " % (binPath,dbname,tables,per_table_num,vgroups,replica)) + + def sql_query_time_cost(self,sql): + startTime = time.time() + tdSql.query(sql) + endTime = time.time() + tdLog.info("sql:%s query time cost (%d)s" % (sql,endTime - startTime)) + + def sql_limit_retun_n_slimit_return_error(self,sql,num,tables,per_table_num,base_fun,replace_fun): + #sql limit n = n;sql limit 0 = 0 ;sql slmit n = error;sql slimit 0 = error + sql = sql.replace('%s'%base_fun,'%s'%replace_fun) + + nest_sql =" select * from (%s) " %sql + self.sql_query_time_cost(sql) + tdSql.checkRows(num) + self.sql_query_time_cost(nest_sql) + tdSql.checkRows(num) + + sql_0 = re.sub(r'\d\d',"0",sql) # \d是匹配数字字符[0-9]，+匹配一个或多个 + self.sql_query_time_cost(sql_0) + tdSql.checkRows(0) + + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) # \d是匹配数字字符[0-9]，+匹配一个或多个 + self.sql_query_time_cost(nest_sql_0) + tdSql.checkRows(0) + + sql = sql.replace('limit','slimit') + tdSql.error(sql) + nest_sql = nest_sql.replace('limit','slimit') + tdSql.error(nest_sql) + + sql_0 = re.sub(r'\d\d',"0",sql) + tdSql.error(sql_0) + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) + tdSql.error(nest_sql_0) + + def sql_data_limit_retun_n_slimit_return_error(self,sql,num,tables,per_table_num,base_fun,replace_fun): + #sql limit n = n;sql limit 0 = 0 ;sql slmit n = error;sql slimit 0 = error + sql = sql.replace('%s'%base_fun,'%s'%replace_fun) + + nest_sql =" select * from (%s) " %sql + self.sql_query_time_cost(sql) + tdSql.checkData(0,0,num) + self.sql_query_time_cost(nest_sql) + tdSql.checkData(0,0,num) + + sql_0 = re.sub(r'\d\d',"0",sql) # \d是匹配数字字符[0-9]，+匹配一个或多个 + self.sql_query_time_cost(sql_0) + tdSql.checkData(0,0,0) + + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) # \d是匹配数字字符[0-9]，+匹配一个或多个 + self.sql_query_time_cost(nest_sql_0) + tdSql.checkData(0,0,0) + + sql = sql.replace('limit','slimit') + tdSql.error(sql) + nest_sql = nest_sql.replace('limit','slimit') + tdSql.error(nest_sql) + + sql_0 = re.sub(r'\d\d',"0",sql) + tdSql.error(sql_0) + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) + tdSql.error(nest_sql_0) + + + def sql_limit_retun_1_slimit_return_error(self,sql,tables,per_table_num,base_fun,replace_fun): + #sql limit n =1;sql limit 0 = 0 ;sql slmit n = error;sql slimit 0 = error + sql = sql.replace('%s'%base_fun,'%s'%replace_fun) + + nest_sql =" select * from (%s) " %sql + self.sql_query_time_cost(sql) + tdSql.checkRows(1) + self.sql_query_time_cost(nest_sql) + tdSql.checkRows(1) + + sql_0 = re.sub(r'\d+',"0",sql) + self.sql_query_time_cost(sql_0) + tdSql.checkRows(0) + + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) # \d是匹配数字字符[0-9]，+匹配一个或多个 + self.sql_query_time_cost(nest_sql_0) + tdSql.checkRows(0) + + sql = sql.replace('limit','slimit') + tdSql.error(sql) + nest_sql = nest_sql.replace('limit','slimit') + tdSql.error(nest_sql) + + sql_0 = re.sub(r'\d+',"0",sql) + tdSql.error(sql_0) + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) + tdSql.error(nest_sql_0) + + def sql_data_limit_retun_1_slimit_return_error(self,sql,tables,per_table_num,base_fun,replace_fun): + #sql limit n =1;sql limit 0 = 0 ;sql slmit n = error;sql slimit 0 = error + sql = sql.replace('%s'%base_fun,'%s'%replace_fun) + + nest_sql =" select * from (%s) " %sql + self.sql_query_time_cost(sql) + tdSql.checkData(0,0,1) + self.sql_query_time_cost(nest_sql) + tdSql.checkData(0,0,1) + + sql_0 = re.sub(r'\d+',"0",sql) + self.sql_query_time_cost(sql_0) + tdSql.checkData(0,0,0) + + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) # \d是匹配数字字符[0-9]，+匹配一个或多个 + self.sql_query_time_cost(nest_sql_0) + tdSql.checkData(0,0,0) + + sql = sql.replace('limit','slimit') + tdSql.error(sql) + nest_sql = nest_sql.replace('limit','slimit') + tdSql.error(nest_sql) + + sql_0 = re.sub(r'\d+',"0",sql) + tdSql.error(sql_0) + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) + tdSql.error(nest_sql_0) + + def sql_last_limit_retun_1_slimit_return_error(self,sql,tables,per_table_num,base_fun,replace_fun): + #sql limit n =1;sql limit 0 = 0 ;sql slmit n = error;sql slimit 0 = error + sql = sql.replace('%s'%base_fun,'%s'%replace_fun) + + nest_sql =" select * from (%s) " %sql + self.sql_query_time_cost(sql) + rows = tdSql.queryRows + if (rows >= 1 or rows <= 4): + tdLog.info("sql checkrows success") + else: + tdLog.exit(f"checkEqual error, sql_rows=={rows}") + + + self.sql_query_time_cost(nest_sql) + rows = tdSql.queryRows + if (rows >= 1 or rows <= 4): + tdLog.info("sql checkrows success") + else: + tdLog.exit(f"checkEqual error, sql_rows=={rows}") + + sql_0 = re.sub(r'\d+',"0",sql) + self.sql_query_time_cost(sql_0) + tdSql.checkRows(0) + + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) # \d是匹配数字字符[0-9]，+匹配一个或多个 + self.sql_query_time_cost(nest_sql_0) + tdSql.checkRows(0) + + sql = sql.replace('limit','slimit') + tdSql.error(sql) + nest_sql = nest_sql.replace('limit','slimit') + tdSql.error(nest_sql) + + sql_0 = re.sub(r'\d+',"0",sql) + tdSql.error(sql_0) + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) + tdSql.error(nest_sql_0) + + def sql_limit_retun_tables_slimit_return_error(self,sql,num,tables,per_table_num,base_fun,replace_fun): + #sql limit n =n;sql limit 0 = 0 ;sql slmit n = error;sql slimit 0 = error + sql = sql.replace('%s'%base_fun,'%s'%replace_fun) + + nest_sql =" select * from (%s) " %sql + self.sql_query_time_cost(sql) + tdSql.checkRows(tables) + self.sql_query_time_cost(nest_sql) + tdSql.checkRows(tables) + + sql_0 = re.sub(r'\d\d',"0",sql) + self.sql_query_time_cost(sql_0) + tdSql.checkRows(0) + + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) # \d是匹配数字字符[0-9]，+匹配一个或多个 + self.sql_query_time_cost(nest_sql_0) + tdSql.checkRows(0) + + sql = sql.replace('limit','slimit') + tdSql.error(sql) + nest_sql = nest_sql.replace('limit','slimit') + tdSql.error(nest_sql) + + sql_0 = re.sub(r'\d+',"0",sql) + tdSql.error(sql_0) + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) + tdSql.error(nest_sql_0) + + def sql_limit_retun_tables_slimit_return_n(self,sql,num,tables,per_table_num,base_fun,replace_fun): + #sql limit n =tables;sql limit 0 = 0 ;sql slmit n = n;sql slimit 0 = 0 + sql = sql.replace('%s'%base_fun,'%s'%replace_fun) + + nest_sql =" select * from (%s) " %sql + self.sql_query_time_cost(sql) + tdSql.checkRows(tables) + self.sql_query_time_cost(nest_sql) + tdSql.checkRows(tables) + + sql_0 = re.sub(r'\d\d',"0",sql) + self.sql_query_time_cost(sql_0) + tdSql.checkRows(0) + + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) # \d是匹配数字字符[0-9]，+匹配一个或多个 + self.sql_query_time_cost(nest_sql_0) + tdSql.checkRows(0) + + sql = sql.replace('limit','slimit') + self.sql_query_time_cost(sql) + tdSql.checkRows(num) + nest_sql = nest_sql.replace('limit','slimit') + self.sql_query_time_cost(nest_sql) + tdSql.checkRows(num) + + sql_0 = re.sub(r'\d\d',"0",sql) + self.sql_query_time_cost(sql_0) + tdSql.checkRows(0) + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) + self.sql_query_time_cost(nest_sql_0) + tdSql.checkRows(0) + + def sql_data_limit_retun_tables_slimit_return_n(self,sql,num,tables,per_table_num,base_fun,replace_fun): + #sql limit n =tables;sql limit 0 = 0 ;sql slmit n = n;sql slimit 0 = 0 + sql = sql.replace('%s'%base_fun,'%s'%replace_fun) + + nest_sql =" select * from (%s) " %sql + self.sql_query_time_cost(sql) + tdSql.checkData(0,0,tables) + self.sql_query_time_cost(nest_sql) + tdSql.checkData(0,0,tables) + + sql_0 = re.sub(r'\d\d',"0",sql) + self.sql_query_time_cost(sql_0) + tdSql.checkData(0,0,0) + + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) # \d是匹配数字字符[0-9]，+匹配一个或多个 + self.sql_query_time_cost(nest_sql_0) + tdSql.checkData(0,0,0) + + sql = sql.replace('limit','slimit') + self.sql_query_time_cost(sql) + tdSql.checkData(0,0,num) + nest_sql = nest_sql.replace('limit','slimit') + self.sql_query_time_cost(nest_sql) + tdSql.checkData(0,0,num) + + sql_0 = re.sub(r'\d\d',"0",sql) + self.sql_query_time_cost(sql_0) + tdSql.checkData(0,0,0) + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) + self.sql_query_time_cost(nest_sql_0) + tdSql.checkData(0,0,0) + + def sql_limit_retun_n_slimit_return_tables(self,sql,num,tables,per_table_num,base_fun,replace_fun): + #sql limit n =n;sql limit 0 = 0 ;sql slmit n = 100;sql slimit 0 = 0 + sql = sql.replace('%s'%base_fun,'%s'%replace_fun) + + nest_sql =" select * from (%s) " %sql + self.sql_query_time_cost(sql) + tdSql.checkRows(num) + self.sql_query_time_cost(nest_sql) + tdSql.checkRows(num) + + sql_0 = re.sub(r'\d\d',"0",sql) + self.sql_query_time_cost(sql_0) + tdSql.checkRows(0) + + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) # \d是匹配数字字符[0-9]，+匹配一个或多个 + self.sql_query_time_cost(nest_sql_0) + tdSql.checkRows(0) + + sql = sql.replace('limit','slimit') + self.sql_query_time_cost(sql) + tdSql.checkRows(tables) + nest_sql = nest_sql.replace('limit','slimit') + self.sql_query_time_cost(nest_sql) + tdSql.checkRows(tables) + + sql_0 = re.sub(r'\d\d',"0",sql) + self.sql_query_time_cost(sql_0) + tdSql.checkRows(0) + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) + self.sql_query_time_cost(nest_sql_0) + tdSql.checkRows(0) + + def sql_data_limit_retun_n_slimit_return_tables(self,sql,num,tables,per_table_num,base_fun,replace_fun): + #sql limit n =n;sql limit 0 = 0 ;sql slmit n = 100;sql slimit 0 = 0 + sql = sql.replace('%s'%base_fun,'%s'%replace_fun) + + nest_sql =" select * from (%s) " %sql + self.sql_query_time_cost(sql) + tdSql.checkData(0,0,num) + self.sql_query_time_cost(nest_sql) + tdSql.checkData(0,0,num) + + sql_0 = re.sub(r'\d\d',"0",sql) + self.sql_query_time_cost(sql_0) + tdSql.checkData(0,0,0) + + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) # \d是匹配数字字符[0-9]，+匹配一个或多个 + self.sql_query_time_cost(nest_sql_0) + tdSql.checkData(0,0,0) + + sql = sql.replace('limit','slimit') + self.sql_query_time_cost(sql) + tdSql.checkData(0,0,tables) + nest_sql = nest_sql.replace('limit','slimit') + self.sql_query_time_cost(nest_sql) + tdSql.checkData(0,0,tables) + + sql_0 = re.sub(r'\d\d',"0",sql) + self.sql_query_time_cost(sql_0) + tdSql.checkData(0,0,0) + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) + self.sql_query_time_cost(nest_sql_0) + tdSql.checkData(0,0,0) + + def sql_limit_retun_tables_times_n_slimit_return_error(self,sql,num,tables,per_table_num,base_fun,replace_fun): + #sql limit n =n*tables;sql limit 0 = 0 ;sql slmit n = tables*n;sql slimit 0 = 0 + #interval + sql = sql.replace('%s'%base_fun,'%s'%replace_fun) + + nest_sql =" select * from (%s) " %sql + self.sql_query_time_cost(sql) + tdSql.checkRows(num*tables) + self.sql_query_time_cost(nest_sql) + tdSql.checkRows(num*tables) + + sql_0 = re.sub(r'\d\d',"0",sql) + self.sql_query_time_cost(sql_0) + tdSql.checkRows(0) + + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) # \d是匹配数字字符[0-9]，+匹配一个或多个 + self.sql_query_time_cost(nest_sql_0) + tdSql.checkRows(0) + + sql = sql.replace('limit','slimit') + tdSql.error(sql) + nest_sql = nest_sql.replace('limit','slimit') + tdSql.error(nest_sql) + + sql_0 = re.sub(r'\d\d',"0",sql) + tdSql.error(sql_0) + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) + tdSql.error(nest_sql_0) + + def sql_limit_retun_tables_times_n_slimit_return_per_table_num_times_n(self,sql,num,tables,per_table_num,base_fun,replace_fun): + #sql limit n =n*tables;sql limit 0 = 0 ;sql slmit n = per_table_num*n;sql slimit 0 = 0 + #interval + sql = sql.replace('%s'%base_fun,'%s'%replace_fun) + + nest_sql =" select * from (%s) " %sql + self.sql_query_time_cost(sql) + tdSql.checkRows(num*tables) + self.sql_query_time_cost(nest_sql) + tdSql.checkRows(num*tables) + + sql_0 = re.sub(r'\d\d',"0",sql) + self.sql_query_time_cost(sql_0) + tdSql.checkRows(0) + + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) # \d是匹配数字字符[0-9]，+匹配一个或多个 + self.sql_query_time_cost(nest_sql_0) + tdSql.checkRows(0) + + sql = sql.replace('limit','slimit') + self.sql_query_time_cost(sql) + tdSql.checkRows(num*per_table_num) + nest_sql = nest_sql.replace('limit','slimit') + self.sql_query_time_cost(nest_sql) + tdSql.checkRows(num*per_table_num) + + sql_0 = re.sub(r'\d\d',"0",sql) + self.sql_query_time_cost(sql_0) + tdSql.checkRows(0) + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) + self.sql_query_time_cost(nest_sql_0) + tdSql.checkRows(0) + + def sql_data_limit_retun_tables_times_n_slimit_return_per_table_num_times_n(self,sql,num,tables,per_table_num,base_fun,replace_fun): + #sql limit n =n*tables;sql limit 0 = 0 ;sql slmit n = per_table_num*n;sql slimit 0 = 0 + #interval + sql = sql.replace('%s'%base_fun,'%s'%replace_fun) + + nest_sql =" select * from (%s) " %sql + self.sql_query_time_cost(sql) + tdSql.checkData(0,0,num*tables) + self.sql_query_time_cost(nest_sql) + tdSql.checkData(0,0,num*tables) + + sql_0 = re.sub(r'\d\d',"0",sql) + self.sql_query_time_cost(sql_0) + tdSql.checkData(0,0,0) + + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) # \d是匹配数字字符[0-9]，+匹配一个或多个 + self.sql_query_time_cost(nest_sql_0) + tdSql.checkData(0,0,0) + + sql = sql.replace('limit','slimit') + self.sql_query_time_cost(sql) + tdSql.checkData(0,0,num*per_table_num) + nest_sql = nest_sql.replace('limit','slimit') + self.sql_query_time_cost(nest_sql) + tdSql.checkData(0,0,num*per_table_num) + + sql_0 = re.sub(r'\d\d',"0",sql) + self.sql_query_time_cost(sql_0) + tdSql.checkData(0,0,0) + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) + self.sql_query_time_cost(nest_sql_0) + tdSql.checkData(0,0,0) + + def sql_limit_retun_n_slimit_return_per_table_num_times_tables(self,sql,num,tables,per_table_num,base_fun,replace_fun): + #sql limit n =n*tables;sql limit 0 = 0 ;sql slmit n = per_table_num*n;sql slimit 0 = 0 + #interval + sql = sql.replace('%s'%base_fun,'%s'%replace_fun) + + nest_sql =" select * from (%s) " %sql + self.sql_query_time_cost(sql) + tdSql.checkRows(num) + self.sql_query_time_cost(nest_sql) + tdSql.checkRows(num) + + sql_0 = re.sub(r'\d\d',"0",sql) + self.sql_query_time_cost(sql_0) + tdSql.checkRows(0) + + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) # \d是匹配数字字符[0-9]，+匹配一个或多个 + self.sql_query_time_cost(nest_sql_0) + tdSql.checkRows(0) + + sql = sql.replace('limit','slimit') + self.sql_query_time_cost(sql) + tdSql.checkRows(tables*per_table_num) + nest_sql = nest_sql.replace('limit','slimit') + self.sql_query_time_cost(nest_sql) + tdSql.checkRows(tables*per_table_num) + + sql_0 = re.sub(r'\d\d',"0",sql) + self.sql_query_time_cost(sql_0) + tdSql.checkRows(0) + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) + self.sql_query_time_cost(nest_sql_0) + tdSql.checkRows(0) + + def sql_data_limit_retun_n_slimit_return_per_table_num_times_tables(self,sql,num,tables,per_table_num,base_fun,replace_fun): + #sql limit n =n*tables;sql limit 0 = 0 ;sql slmit n = per_table_num*n;sql slimit 0 = 0 + #interval + sql = sql.replace('%s'%base_fun,'%s'%replace_fun) + + nest_sql =" select * from (%s) " %sql + self.sql_query_time_cost(sql) + tdSql.checkData(0,0,num) + self.sql_query_time_cost(nest_sql) + tdSql.checkData(0,0,num) + + sql_0 = re.sub(r'\d\d',"0",sql) + self.sql_query_time_cost(sql_0) + tdSql.checkData(0,0,0) + + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) # \d是匹配数字字符[0-9]，+匹配一个或多个 + self.sql_query_time_cost(nest_sql_0) + tdSql.checkData(0,0,0) + + sql = sql.replace('limit','slimit') + self.sql_query_time_cost(sql) + tdSql.checkData(0,0,tables*per_table_num) + nest_sql = nest_sql.replace('limit','slimit') + self.sql_query_time_cost(nest_sql) + tdSql.checkData(0,0,tables*per_table_num) + + sql_0 = re.sub(r'\d\d',"0",sql) + self.sql_query_time_cost(sql_0) + tdSql.checkData(0,0,0) + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) + self.sql_query_time_cost(nest_sql_0) + tdSql.checkData(0,0,0) + + def sql_limit_not_test_slimitkeep_return_per_table_num_times_n(self,sql,num,tables,per_table_num,base_fun,replace_fun): + #sql sql slmit n = per_table_num*n;sql slimit 0 = 0 + sql = sql.replace('%s'%base_fun,'%s'%replace_fun) + + nest_sql =" select * from (%s) " %sql + sql = sql.replace('limit','limit') + self.sql_query_time_cost(sql) + tdSql.checkRows(num*per_table_num) + + self.sql_query_time_cost(nest_sql) + tdSql.checkRows(num*per_table_num) + + sql_0 = re.sub(r'\d\d',"0",sql) + self.sql_query_time_cost(sql_0) + tdSql.checkRows(0) + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) + self.sql_query_time_cost(nest_sql_0) + tdSql.checkRows(0) + + def sql_data_limit_not_test_slimitkeep_return_per_table_num_times_n(self,sql,num,tables,per_table_num,base_fun,replace_fun): + #sql sql slmit n = per_table_num*n;sql slimit 0 = 0 + sql = sql.replace('%s'%base_fun,'%s'%replace_fun) + + nest_sql =" select * from (%s) " %sql + sql = sql.replace('limit','limit') + self.sql_query_time_cost(sql) + tdSql.checkData(0,0,num*per_table_num) + + self.sql_query_time_cost(nest_sql) + tdSql.checkData(0,0,num*per_table_num) + + sql_0 = re.sub(r'\d\d',"0",sql) + self.sql_query_time_cost(sql_0) + tdSql.checkData(0,0,0) + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) + self.sql_query_time_cost(nest_sql_0) + tdSql.checkData(0,0,0) + + def sql_limit_not_test_slimitkeep_return_per_table_num_times_tables(self,sql,num,tables,per_table_num,base_fun,replace_fun): + #sql sql slmit n = per_table_num*tables;sql slimit 0 = 0 + sql = sql.replace('%s'%base_fun,'%s'%replace_fun) + + nest_sql =" select * from (%s) " %sql + self.sql_query_time_cost(sql) + tdSql.checkRows(tables*per_table_num) + + self.sql_query_time_cost(nest_sql) + tdSql.checkRows(tables*per_table_num) + + sql_0 = re.sub(r'\d\d',"0",sql) + self.sql_query_time_cost(sql_0) + tdSql.checkRows(0) + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) + self.sql_query_time_cost(nest_sql_0) + tdSql.checkRows(0) + + def sql_data_limit_not_test_slimitkeep_return_per_table_num_times_tables(self,sql,num,tables,per_table_num,base_fun,replace_fun): + #sql sql slmit n = per_table_num*tables;sql slimit 0 = 0 + sql = sql.replace('%s'%base_fun,'%s'%replace_fun) + + nest_sql =" select * from (%s) " %sql + self.sql_query_time_cost(sql) + tdSql.checkData(0,0,tables*per_table_num) + + self.sql_query_time_cost(nest_sql) + tdSql.checkData(0,0,tables*per_table_num) + + sql_0 = re.sub(r'\d\d',"0",sql) + self.sql_query_time_cost(sql_0) + tdSql.checkData(0,0,0) + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) + self.sql_query_time_cost(nest_sql_0) + tdSql.checkData(0,0,0) + + def sql_limit_retun_n_times_n2_slimitkeep_return_n_times_n2(self,sql,num,num2,tables,per_table_num,base_fun,replace_fun): + #sql limit n =n*num2;sql limit 0 = 0 ;sql slmit n = num2*n;sql slimit 0 = 0 + sql = sql.replace('%s'%base_fun,'%s'%replace_fun) + + nest_sql =" select * from (%s) " %sql + self.sql_query_time_cost(sql) + tdSql.checkRows(num*num2) + self.sql_query_time_cost(nest_sql) + tdSql.checkRows(num*num2) + + sql_0 = re.sub(r'\d\d',"0",sql) + self.sql_query_time_cost(sql_0) + tdSql.checkRows(0) + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) + self.sql_query_time_cost(nest_sql_0) + tdSql.checkRows(0) + + sql = sql.replace('limit','limit') + self.sql_query_time_cost(sql) + tdSql.checkRows(num*num2) + nest_sql = nest_sql.replace('limit','limit') + self.sql_query_time_cost(nest_sql) + tdSql.checkRows(num*num2) + + sql_0 = re.sub(r'\d\d',"0",sql) + self.sql_query_time_cost(sql_0) + tdSql.checkRows(0) + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) + self.sql_query_time_cost(nest_sql_0) + tdSql.checkRows(0) + + def sql_data_limit_retun_n_times_n2_slimitkeep_return_n_times_n2(self,sql,num,num2,tables,per_table_num,base_fun,replace_fun): + #sql limit n =n*num2;sql limit 0 = 0 ;sql slmit n = num2*n;sql slimit 0 = 0 + sql = sql.replace('%s'%base_fun,'%s'%replace_fun) + + nest_sql =" select * from (%s) " %sql + self.sql_query_time_cost(sql) + tdSql.checkData(0,0,num*num2) + self.sql_query_time_cost(nest_sql) + tdSql.checkData(0,0,num*num2) + + sql_0 = re.sub(r'\d\d',"0",sql) + self.sql_query_time_cost(sql_0) + tdSql.checkData(0,0,0) + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) + self.sql_query_time_cost(nest_sql_0) + tdSql.checkData(0,0,0) + + sql = sql.replace('limit','limit') + self.sql_query_time_cost(sql) + tdSql.checkData(0,0,num*num2) + nest_sql = nest_sql.replace('limit','limit') + self.sql_query_time_cost(nest_sql) + tdSql.checkData(0,0,num*num2) + + sql_0 = re.sub(r'\d\d',"0",sql) + self.sql_query_time_cost(sql_0) + tdSql.checkData(0,0,0) + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) + self.sql_query_time_cost(nest_sql_0) + tdSql.checkData(0,0,0) + + def sql_limit_times_slimitkeep_return_n2(self,sql,num,num2,tables,per_table_num,base_fun,replace_fun): + #sql limit n =n*num2;sql limit 0 = 0 ;sql slmit n = num2*n;sql slimit 0 = 0 + sql = sql.replace('%s'%base_fun,'%s'%replace_fun) + + nest_sql =" select * from (%s) " %sql + self.sql_query_time_cost(sql) + tdSql.checkRows(num2) + self.sql_query_time_cost(nest_sql) + tdSql.checkRows(num2) + + sql_0 = re.sub(r'\d\d',"0",sql) + self.sql_query_time_cost(sql_0) + tdSql.checkRows(0) + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) + self.sql_query_time_cost(nest_sql_0) + tdSql.checkRows(0) + + def sql_data_limit_times_slimitkeep_return_n2(self,sql,num,num2,tables,per_table_num,base_fun,replace_fun): + #sql limit n =n*num2;sql limit 0 = 0 ;sql slmit n = num2*n;sql slimit 0 = 0 + sql = sql.replace('%s'%base_fun,'%s'%replace_fun) + + nest_sql =" select * from (%s) " %sql + self.sql_query_time_cost(sql) + tdSql.checkData(0,0,num2) + self.sql_query_time_cost(nest_sql) + tdSql.checkData(0,0,num2) + + sql_0 = re.sub(r'\d\d',"0",sql) + self.sql_query_time_cost(sql_0) + tdSql.checkData(0,0,0) + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) + self.sql_query_time_cost(nest_sql_0) + tdSql.checkData(0,0,0) + + def sql_retun_error(self,sql,base_fun,replace_fun): + #sql limit n = error;sql limit 0 = error ;sql slmit n = error ;sql slimit 0 = error + sql = sql.replace('%s'%base_fun,'%s'%replace_fun) + + nest_sql =" select * from (%s) " %sql + tdSql.error(sql) + tdSql.error(nest_sql) + + sql_0 = re.sub(r'\d+',"0",sql) + tdSql.error(sql) + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) + tdSql.error(nest_sql_0) + + sql = sql.replace('limit','slimit') + tdSql.error(sql) + nest_sql = nest_sql.replace('limit','slimit') + tdSql.error(nest_sql) + + sql_0 = re.sub(r'\d+',"0",sql) + tdSql.error(sql_0) + nest_sql_0 = re.sub(r'\d\d',"0",nest_sql) + tdSql.error(nest_sql_0) + + def fun_base(self,dbname,num,num2,tables,per_table_num,dbnamejoin,base_fun,replace_fun): + + tdLog.info("base query ---------1----------") + sql = "select * from %s.meters limit %d" %(dbname,num) + self.sql_limit_retun_n_slimit_return_error(sql,num,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_retun_n_slimit_return_error(sql,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_retun_n_slimit_return_error(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_retun_n_slimit_return_error(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select a.* from %s.meters a,%s.meters b where a.ts = b.ts limit %d" %(dbname,dbnamejoin,num) + self.sql_limit_retun_n_slimit_return_error(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_retun_n_slimit_return_error(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_retun_n_slimit_return_error(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_retun_n_slimit_return_error(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + + + tdLog.info("base query ---------2----------") + sql = "select * from %s.meters where ts is not null limit %d" %(dbname,num) + self.sql_limit_retun_n_slimit_return_error(sql,num,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_retun_n_slimit_return_error(sql,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_retun_n_slimit_return_error(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_retun_n_slimit_return_error(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select a.* from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts limit %d" %(dbname,dbnamejoin,num) + self.sql_limit_retun_n_slimit_return_error(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_retun_n_slimit_return_error(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_retun_n_slimit_return_error(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_retun_n_slimit_return_error(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + + + tdLog.info("base query ---------3----------") + sql = "select * from %s.meters where ts is not null order by ts limit %d" %(dbname,num) + self.sql_limit_retun_n_slimit_return_error(sql,num,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_retun_n_slimit_return_error(sql,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_retun_n_slimit_return_error(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_retun_n_slimit_return_error(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select a.* from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts order by b.ts limit %d" %(dbname,dbnamejoin,num) + self.sql_limit_retun_n_slimit_return_error(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_retun_n_slimit_return_error(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_retun_n_slimit_return_error(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_retun_n_slimit_return_error(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + + tdLog.info("base query ---------4----------") + sql = "select * from %s.meters where ts is not null order by ts desc limit %d" %(dbname,num) + self.sql_limit_retun_n_slimit_return_error(sql,num,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_retun_n_slimit_return_error(sql,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_retun_n_slimit_return_error(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_retun_n_slimit_return_error(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select a.* from %s.meters a,%s.meters b where b.ts is not null and a.ts = b.ts order by a.ts desc limit %d" %(dbname,dbnamejoin,num) + self.sql_limit_retun_n_slimit_return_error(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_retun_n_slimit_return_error(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_retun_n_slimit_return_error(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_retun_n_slimit_return_error(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + def fun_count(self,dbname,num,num2,tables,per_table_num,dbnamejoin,base_fun,replace_fun): + + tdLog.info("count query ---------1----------") + sql = "select count(*) from %s.meters limit %d" %(dbname,num) + self.sql_limit_retun_1_slimit_return_error(sql,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_retun_1_slimit_return_error(sql,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_retun_1_slimit_return_error(sql_union,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_retun_1_slimit_return_error(sql_union_all,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select count(a.*) from %s.meters a,%s.meters b where a.ts = b.ts limit %d" %(dbname,dbnamejoin,num) + self.sql_limit_retun_1_slimit_return_error(sql_join,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_retun_1_slimit_return_error(sql_join,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_retun_1_slimit_return_error(sql_union,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_retun_1_slimit_return_error(sql_union_all,tables,per_table_num,base_fun,replace_fun) + + + + + tdLog.info("count query ---------2----------") + sql = "select count(*) from %s.meters where ts is not null limit %d" %(dbname,num) + self.sql_limit_retun_1_slimit_return_error(sql,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_retun_1_slimit_return_error(sql,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_retun_1_slimit_return_error(sql_union,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_retun_1_slimit_return_error(sql_union_all,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select count(a.*) from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts limit %d" %(dbname,dbnamejoin,num) + self.sql_limit_retun_1_slimit_return_error(sql_join,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_retun_1_slimit_return_error(sql_join,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_retun_1_slimit_return_error(sql_union,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_retun_1_slimit_return_error(sql_union_all,tables,per_table_num,base_fun,replace_fun) + + + + tdLog.info("count query ---------3----------") + sql = "select count(*) from %s.meters where ts is not null order by ts limit %d" %(dbname,num) + self.sql_retun_error(sql,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_retun_error(sql,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_retun_error(sql_union,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_retun_error(sql_union_all,base_fun,replace_fun) + + sql_join = "select count(a.*) from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts order by b.ts limit %d" %(dbname,dbnamejoin,num) + self.sql_retun_error(sql_join,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_retun_error(sql_join,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_retun_error(sql_union,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_retun_error(sql_union_all,base_fun,replace_fun) + + + + tdLog.info("count query ---------4----------") + sql = "select count(*) from %s.meters where ts is not null order by ts desc limit %d" %(dbname,num) + self.sql_retun_error(sql,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_retun_error(sql,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_retun_error(sql_union,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_retun_error(sql_union_all,base_fun,replace_fun) + + sql_join = "select count(a.*) from %s.meters a,%s.meters b where b.ts is not null and a.ts = b.ts order by a.ts desc limit %d" %(dbname,dbnamejoin,num) + self.sql_retun_error(sql_join,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_retun_error(sql_join,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_retun_error(sql_union,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_retun_error(sql_union_all,base_fun,replace_fun) + + + + tdLog.info("count query ---------5----------") + sql = "select count(*) from %s.meters where ts is not null group by tbname limit %d" %(dbname,num) + self.sql_limit_retun_tables_slimit_return_n(sql,num,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_retun_tables_slimit_return_n(sql,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_retun_tables_slimit_return_n(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_retun_tables_slimit_return_n(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select count(*) from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts group by b.tbname limit %d" %(dbname,dbnamejoin,num) + self.sql_limit_retun_tables_slimit_return_n(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_retun_tables_slimit_return_n(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_retun_tables_slimit_return_n(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_retun_tables_slimit_return_n(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + + + + tdLog.info("count query ---------6----------") + sql = "select count(*) from %s.meters where ts is not null partition by tbname limit %d" %(dbname,num) + self.sql_limit_retun_tables_slimit_return_n(sql,num,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_retun_tables_slimit_return_n(sql,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_retun_tables_slimit_return_n(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_retun_tables_slimit_return_n(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select count(*) from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts partition by b.tbname limit %d" %(dbname,dbnamejoin,num) + self.sql_limit_retun_tables_slimit_return_n(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_retun_tables_slimit_return_n(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_retun_tables_slimit_return_n(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_retun_tables_slimit_return_n(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + + + tdLog.info("count query ---------7----------") + sql = "select count(*) cc from %s.meters where ts is not null group by tbname order by cc limit %d" %(dbname,num) + self.sql_limit_retun_n_slimit_return_tables(sql,num,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_retun_n_slimit_return_tables(sql,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_retun_n_slimit_return_tables(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_retun_n_slimit_return_tables(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select count(*) cc from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts group by b.tbname order by cc limit %d" %(dbname,dbnamejoin,num) + self.sql_limit_retun_n_slimit_return_tables(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_retun_n_slimit_return_tables(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_retun_n_slimit_return_tables(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_retun_n_slimit_return_tables(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + + + tdLog.info("count query ---------8----------") + sql = "select count(*) cc from %s.meters where ts is not null partition by tbname order by cc limit %d" %(dbname,num) + self.sql_limit_retun_n_slimit_return_tables(sql,num,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_retun_n_slimit_return_tables(sql,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_retun_n_slimit_return_tables(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_retun_n_slimit_return_tables(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select count(*) cc from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts partition by b.tbname order by cc limit %d" %(dbname,dbnamejoin,num) + self.sql_limit_retun_n_slimit_return_tables(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_retun_n_slimit_return_tables(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_retun_n_slimit_return_tables(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_retun_n_slimit_return_tables(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + + + tdLog.info("count query ---------9----------") + sql = "select count(*) cc from %s.meters where ts is not null interval(1a) limit %d" %(dbname,num) + self.sql_limit_retun_n_slimit_return_error(sql,num,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_retun_n_slimit_return_error(sql,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_retun_n_slimit_return_error(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_retun_n_slimit_return_error(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select count(*) cc from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts interval(1a) limit %d" %(dbname,dbnamejoin,num) + self.sql_limit_retun_n_slimit_return_error(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_retun_n_slimit_return_error(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_retun_n_slimit_return_error(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_retun_n_slimit_return_error(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + + + tdLog.info("count query ---------10----------") + sql = "select count(*) cc from %s.meters where ts is not null interval(1a) order by cc asc limit %d" %(dbname,num) + self.sql_limit_retun_n_slimit_return_error(sql,num,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_retun_n_slimit_return_error(sql,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_retun_n_slimit_return_error(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_retun_n_slimit_return_error(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select count(*) cc from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts interval(1a) order by cc asc limit %d" %(dbname,dbnamejoin,num) + self.sql_limit_retun_n_slimit_return_error(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_retun_n_slimit_return_error(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_retun_n_slimit_return_error(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_retun_n_slimit_return_error(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + + + tdLog.info("count query ---------11----------") + sql = "select count(*) cc from %s.meters where ts is not null interval(1a) order by cc desc limit %d" %(dbname,num) + self.sql_limit_retun_n_slimit_return_error(sql,num,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_retun_n_slimit_return_error(sql,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_retun_n_slimit_return_error(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_retun_n_slimit_return_error(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select count(*) cc from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts interval(1a) order by cc desc limit %d" %(dbname,dbnamejoin,num) + self.sql_limit_retun_n_slimit_return_error(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_retun_n_slimit_return_error(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_retun_n_slimit_return_error(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_retun_n_slimit_return_error(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + + + tdLog.info("count query ---------12----------") + sql = "select tbname,count(*) cc from %s.meters where ts is not null interval(1a) group by tbname limit %d" %(dbname,num) + self.sql_retun_error(sql,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_retun_error(sql,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_retun_error(sql_union,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_retun_error(sql_union_all,base_fun,replace_fun) + + sql_join = "select a.tbname,count(*) cc from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts interval(1a) group by b.tbname limit %d" %(dbname,dbnamejoin,num) + self.sql_retun_error(sql_join,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_retun_error(sql_join,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_retun_error(sql_union,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_retun_error(sql_union_all,base_fun,replace_fun) + + + + tdLog.info("count query ---------13----------") + sql = "select tbname,count(*) cc from %s.meters where ts is not null interval(1a) partition by tbname limit %d" %(dbname,num) + self.sql_retun_error(sql,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_retun_error(sql,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_retun_error(sql_union,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_retun_error(sql_union_all,base_fun,replace_fun) + + sql_join = "select a.tbname,count(*) cc from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts interval(1a) partition by b.tbname limit %d" %(dbname,dbnamejoin,num) + self.sql_retun_error(sql_join,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_retun_error(sql_join,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_retun_error(sql_union,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_retun_error(sql_union_all,base_fun,replace_fun) + + + + tdLog.info("count query ---------14----------") + sql = "select tbname,count(*) cc from %s.meters where ts is not null partition by tbname interval(1a) limit %d" %(dbname,num) + self.sql_limit_retun_tables_times_n_slimit_return_per_table_num_times_n(sql,num,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_retun_tables_times_n_slimit_return_per_table_num_times_n(sql,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_retun_tables_times_n_slimit_return_per_table_num_times_n(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_retun_tables_times_n_slimit_return_per_table_num_times_n(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select a.tbname,count(*) cc from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts partition by a.tbname interval(1a) limit %d" %(dbname,dbnamejoin,num) + self.sql_limit_retun_tables_times_n_slimit_return_per_table_num_times_n(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_retun_tables_times_n_slimit_return_per_table_num_times_n(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_retun_tables_times_n_slimit_return_per_table_num_times_n(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_retun_tables_times_n_slimit_return_per_table_num_times_n(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + + + tdLog.info("count query ---------15----------") + sql = "select tbname,count(*) cc from %s.meters where ts is not null partition by tbname interval(1a) order by cc asc limit %d" %(dbname,num) + self.sql_limit_retun_n_slimit_return_per_table_num_times_tables(sql,num,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_retun_n_slimit_return_per_table_num_times_tables(sql,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_retun_n_slimit_return_per_table_num_times_tables(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_retun_n_slimit_return_per_table_num_times_tables(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select a.tbname,count(*) cc from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts partition by a.tbname interval(1a) order by cc asc limit %d" %(dbname,dbnamejoin,num) + self.sql_limit_retun_n_slimit_return_per_table_num_times_tables(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_retun_n_slimit_return_per_table_num_times_tables(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_retun_n_slimit_return_per_table_num_times_tables(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_retun_n_slimit_return_per_table_num_times_tables(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + + + tdLog.info("count query ---------16----------") + sql = "select tbname,count(*) cc from %s.meters where ts is not null partition by tbname interval(1a) order by cc desc limit %d" %(dbname,num) + self.sql_limit_retun_n_slimit_return_per_table_num_times_tables(sql,num,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_retun_n_slimit_return_per_table_num_times_tables(sql,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_retun_n_slimit_return_per_table_num_times_tables(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_retun_n_slimit_return_per_table_num_times_tables(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select a.tbname,count(*) cc from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts partition by a.tbname interval(1a) order by cc desc limit %d" %(dbname,dbnamejoin,num) + self.sql_limit_retun_n_slimit_return_per_table_num_times_tables(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_retun_n_slimit_return_per_table_num_times_tables(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_retun_n_slimit_return_per_table_num_times_tables(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_retun_n_slimit_return_per_table_num_times_tables(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + + + tdLog.info("count query ---------17----------") + sql = "select tbname,count(*) cc from %s.meters where ts is not null partition by tbname interval(1a) slimit %d" %(dbname,num) + self.sql_limit_not_test_slimitkeep_return_per_table_num_times_n(sql,num,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_not_test_slimitkeep_return_per_table_num_times_n(sql,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_not_test_slimitkeep_return_per_table_num_times_n(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_not_test_slimitkeep_return_per_table_num_times_n(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select a.tbname,count(*) cc from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts partition by a.tbname interval(1a) slimit %d" %(dbname,dbnamejoin,num) + self.sql_limit_not_test_slimitkeep_return_per_table_num_times_n(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_not_test_slimitkeep_return_per_table_num_times_n(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_not_test_slimitkeep_return_per_table_num_times_n(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_not_test_slimitkeep_return_per_table_num_times_n(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + + + tdLog.info("count query ---------18----------") + sql = "select tbname,count(*) cc from %s.meters where ts is not null partition by tbname interval(1a) order by cc asc slimit %d" %(dbname,num) + self.sql_limit_not_test_slimitkeep_return_per_table_num_times_tables(sql,num,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_not_test_slimitkeep_return_per_table_num_times_tables(sql,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_not_test_slimitkeep_return_per_table_num_times_tables(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_not_test_slimitkeep_return_per_table_num_times_tables(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select a.tbname,count(*) cc from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts partition by a.tbname interval(1a) order by cc asc slimit %d" %(dbname,dbnamejoin,num) + self.sql_limit_not_test_slimitkeep_return_per_table_num_times_tables(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_not_test_slimitkeep_return_per_table_num_times_tables(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_not_test_slimitkeep_return_per_table_num_times_tables(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_not_test_slimitkeep_return_per_table_num_times_tables(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + + tdLog.info("count query ---------19----------") + sql = "select tbname,count(*) cc from %s.meters where ts is not null partition by tbname interval(1a) order by cc desc slimit %d" %(dbname,num) + self.sql_limit_not_test_slimitkeep_return_per_table_num_times_tables(sql,num,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_not_test_slimitkeep_return_per_table_num_times_tables(sql,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_not_test_slimitkeep_return_per_table_num_times_tables(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_not_test_slimitkeep_return_per_table_num_times_tables(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select a.tbname,count(*) cc from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts partition by a.tbname interval(1a) order by cc desc slimit %d" %(dbname,dbnamejoin,num) + self.sql_limit_not_test_slimitkeep_return_per_table_num_times_tables(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_not_test_slimitkeep_return_per_table_num_times_tables(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_not_test_slimitkeep_return_per_table_num_times_tables(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_not_test_slimitkeep_return_per_table_num_times_tables(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + + + tdLog.info("count query ---------20----------") + sql = "select tbname,count(*) cc from %s.meters where ts is not null partition by tbname interval(1a) slimit %d limit %d" %(dbname,num,num2) + self.sql_limit_retun_n_times_n2_slimitkeep_return_n_times_n2(sql,num,num2,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_retun_n_times_n2_slimitkeep_return_n_times_n2(sql,num,num2,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_retun_n_times_n2_slimitkeep_return_n_times_n2(sql_union,num,num2,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_retun_n_times_n2_slimitkeep_return_n_times_n2(sql_union_all,num,num2,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select a.tbname,count(*) cc from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts partition by a.tbname interval(1a) slimit %d limit %d" %(dbname,dbnamejoin,num,num2) + self.sql_limit_retun_n_times_n2_slimitkeep_return_n_times_n2(sql_join,num,num2,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_retun_n_times_n2_slimitkeep_return_n_times_n2(sql_join,num,num2,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_retun_n_times_n2_slimitkeep_return_n_times_n2(sql_union,num,num2,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_retun_n_times_n2_slimitkeep_return_n_times_n2(sql_union_all,num,num2,tables,per_table_num,base_fun,replace_fun) + + + tdLog.info("count query ---------21----------") + sql = "select tbname,count(*) cc from %s.meters where ts is not null partition by tbname interval(1a) order by cc asc slimit %d limit %d" %(dbname,num,num2) + self.sql_limit_times_slimitkeep_return_n2(sql,num,num2,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_times_slimitkeep_return_n2(sql,num,num2,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_times_slimitkeep_return_n2(sql_union,num,num2,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_times_slimitkeep_return_n2(sql_union_all,num,num2,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select a.tbname,count(*) cc from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts partition by a.tbname interval(1a) order by cc asc slimit %d limit %d" %(dbname,dbnamejoin,num,num2) + self.sql_limit_times_slimitkeep_return_n2(sql_join,num,num2,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_times_slimitkeep_return_n2(sql_join,num,num2,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_times_slimitkeep_return_n2(sql_union,num,num2,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_times_slimitkeep_return_n2(sql_union_all,num,num2,tables,per_table_num,base_fun,replace_fun) + + + tdLog.info("count query ---------22----------") + sql = "select tbname,count(*) cc from %s.meters where ts is not null partition by tbname interval(1a) order by cc desc slimit %d limit %d" %(dbname,num,num2) + self.sql_limit_times_slimitkeep_return_n2(sql,num,num2,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_times_slimitkeep_return_n2(sql,num,num2,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_times_slimitkeep_return_n2(sql_union,num,num2,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_times_slimitkeep_return_n2(sql_union_all,num,num2,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select a.tbname,count(*) cc from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts partition by a.tbname interval(1a) order by cc desc slimit %d limit %d" %(dbname,dbnamejoin,num,num2) + self.sql_limit_times_slimitkeep_return_n2(sql_join,num,num2,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_times_slimitkeep_return_n2(sql_join,num,num2,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_times_slimitkeep_return_n2(sql_union,num,num2,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_times_slimitkeep_return_n2(sql_union_all,num,num2,tables,per_table_num,base_fun,replace_fun) + + + def fun_last(self,dbname,num,num2,tables,per_table_num,dbnamejoin,base_fun,replace_fun): + + tdLog.info("last query ---------1----------") + sql = "select last(*) from %s.meters limit %d" %(dbname,num) + self.sql_limit_retun_1_slimit_return_error(sql,tables,per_table_num,base_fun,replace_fun) + sql = "select last(*) from (%s)" %sql + self.sql_limit_retun_1_slimit_return_error(sql,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_last_limit_retun_1_slimit_return_error(sql_union,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_last_limit_retun_1_slimit_return_error(sql_union_all,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select last(a.*) from %s.meters a,%s.meters b where a.ts = b.ts limit %d" %(dbname,dbnamejoin,num) + self.sql_limit_retun_1_slimit_return_error(sql_join,tables,per_table_num,base_fun,replace_fun) + sql_join = "select last(*) from (%s)" %sql_join + self.sql_limit_retun_1_slimit_return_error(sql_join,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_last_limit_retun_1_slimit_return_error(sql_union,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_last_limit_retun_1_slimit_return_error(sql_union_all,tables,per_table_num,base_fun,replace_fun) + + + + + tdLog.info("last query ---------2----------") + sql = "select last(*) from %s.meters where ts is not null limit %d" %(dbname,num) + self.sql_limit_retun_1_slimit_return_error(sql,tables,per_table_num,base_fun,replace_fun) + sql = "select last(*) from (%s)" %sql + self.sql_limit_retun_1_slimit_return_error(sql,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_last_limit_retun_1_slimit_return_error(sql_union,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_last_limit_retun_1_slimit_return_error(sql_union_all,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select last(a.*) from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts limit %d" %(dbname,dbnamejoin,num) + self.sql_limit_retun_1_slimit_return_error(sql_join,tables,per_table_num,base_fun,replace_fun) + sql_join = "select last(*) from (%s)" %sql_join + self.sql_limit_retun_1_slimit_return_error(sql_join,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_last_limit_retun_1_slimit_return_error(sql_union,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_last_limit_retun_1_slimit_return_error(sql_union_all,tables,per_table_num,base_fun,replace_fun) + + + + tdLog.info("last query ---------3----------") + sql = "select last(*) from %s.meters where ts is not null order by ts limit %d" %(dbname,num) + self.sql_retun_error(sql,base_fun,replace_fun) + sql = "select last(*) from (%s)" %sql + self.sql_retun_error(sql,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_retun_error(sql_union,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_retun_error(sql_union_all,base_fun,replace_fun) + + sql_join = "select last(a.*) from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts order by b.ts limit %d" %(dbname,dbnamejoin,num) + self.sql_retun_error(sql_join,base_fun,replace_fun) + sql_join = "select last(*) from (%s)" %sql_join + self.sql_retun_error(sql_join,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_retun_error(sql_union,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_retun_error(sql_union_all,base_fun,replace_fun) + + + + tdLog.info("last query ---------4----------") + sql = "select last(*) from %s.meters where ts is not null order by ts desc limit %d" %(dbname,num) + self.sql_retun_error(sql,base_fun,replace_fun) + sql = "select last(*) from (%s)" %sql + self.sql_retun_error(sql,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_retun_error(sql_union,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_retun_error(sql_union_all,base_fun,replace_fun) + + sql_join = "select last(a.*) from %s.meters a,%s.meters b where b.ts is not null and a.ts = b.ts order by a.ts desc limit %d" %(dbname,dbnamejoin,num) + self.sql_retun_error(sql_join,base_fun,replace_fun) + sql_join = "select last(*) from (%s)" %sql_join + self.sql_retun_error(sql_join,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_retun_error(sql_union,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_retun_error(sql_union_all,base_fun,replace_fun) + + + + tdLog.info("last query ---------5----------") + sql = "select last(*) from %s.meters where ts is not null group by tbname limit %d" %(dbname,num) + self.sql_limit_retun_tables_slimit_return_n(sql,num,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_retun_tables_slimit_return_n(sql,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_retun_tables_slimit_return_n(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_retun_tables_slimit_return_n(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select last(a.*) from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts group by b.tbname limit %d" %(dbname,dbnamejoin,num) + self.sql_limit_retun_tables_slimit_return_n(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_retun_tables_slimit_return_n(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_retun_tables_slimit_return_n(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_retun_tables_slimit_return_n(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + + + + tdLog.info("last query ---------6----------") + sql = "select last(*) from %s.meters where ts is not null partition by tbname limit %d" %(dbname,num) + self.sql_limit_retun_tables_slimit_return_n(sql,num,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_retun_tables_slimit_return_n(sql,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_retun_tables_slimit_return_n(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_retun_tables_slimit_return_n(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select last(a.*) from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts partition by b.tbname limit %d" %(dbname,dbnamejoin,num) + self.sql_limit_retun_tables_slimit_return_n(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_retun_tables_slimit_return_n(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_retun_tables_slimit_return_n(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_retun_tables_slimit_return_n(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + + + tdLog.info("last query ---------7----------") + sql = "select last(ts) cc from %s.meters where ts is not null group by tbname order by cc limit %d" %(dbname,num) + self.sql_limit_retun_n_slimit_return_tables(sql,num,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_retun_n_slimit_return_tables(sql,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_retun_n_slimit_return_tables(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_retun_n_slimit_return_tables(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select last(a.ts) cc from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts group by b.tbname order by cc limit %d" %(dbname,dbnamejoin,num) + self.sql_limit_retun_n_slimit_return_tables(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_retun_n_slimit_return_tables(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_retun_n_slimit_return_tables(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_retun_n_slimit_return_tables(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + + + tdLog.info("last query ---------8----------") + sql = "select last(ts) cc from %s.meters where ts is not null partition by tbname order by cc limit %d" %(dbname,num) + self.sql_limit_retun_n_slimit_return_tables(sql,num,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_retun_n_slimit_return_tables(sql,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_retun_n_slimit_return_tables(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_retun_n_slimit_return_tables(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select last(a.ts) cc from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts partition by b.tbname order by cc limit %d" %(dbname,dbnamejoin,num) + self.sql_limit_retun_n_slimit_return_tables(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_retun_n_slimit_return_tables(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_retun_n_slimit_return_tables(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_retun_n_slimit_return_tables(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + + + tdLog.info("last query ---------9----------") + sql = "select last(*) from %s.meters where ts is not null interval(1a) limit %d" %(dbname,num) + self.sql_limit_retun_n_slimit_return_error(sql,num,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_retun_n_slimit_return_error(sql,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_retun_n_slimit_return_error(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_retun_n_slimit_return_error(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select last(a.ts) cc from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts interval(1a) limit %d" %(dbname,dbnamejoin,num) + self.sql_limit_retun_n_slimit_return_error(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_retun_n_slimit_return_error(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_retun_n_slimit_return_error(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_retun_n_slimit_return_error(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + + + tdLog.info("last query ---------10----------") + sql = "select last(ts) cc from %s.meters where ts is not null interval(1a) order by cc asc limit %d" %(dbname,num) + self.sql_limit_retun_n_slimit_return_error(sql,num,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_retun_n_slimit_return_error(sql,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_retun_n_slimit_return_error(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_retun_n_slimit_return_error(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select last(a.ts) cc from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts interval(1a) order by cc asc limit %d" %(dbname,dbnamejoin,num) + self.sql_limit_retun_n_slimit_return_error(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_retun_n_slimit_return_error(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_retun_n_slimit_return_error(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_retun_n_slimit_return_error(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + + + tdLog.info("last query ---------11----------") + sql = "select last(ts) cc from %s.meters where ts is not null interval(1a) order by cc desc limit %d" %(dbname,num) + self.sql_limit_retun_n_slimit_return_error(sql,num,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_retun_n_slimit_return_error(sql,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_retun_n_slimit_return_error(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_retun_n_slimit_return_error(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select last(a.ts) cc from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts interval(1a) order by cc desc limit %d" %(dbname,dbnamejoin,num) + self.sql_limit_retun_n_slimit_return_error(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_retun_n_slimit_return_error(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_retun_n_slimit_return_error(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_retun_n_slimit_return_error(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + + + tdLog.info("last query ---------12----------") + sql = "select tbname,last(ts) cc from %s.meters where ts is not null interval(1a) group by tbname limit %d" %(dbname,num) + self.sql_retun_error(sql,base_fun,replace_fun) + sql = "select last(*) from (%s)" %sql + self.sql_retun_error(sql,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_retun_error(sql_union,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_retun_error(sql_union_all,base_fun,replace_fun) + + sql_join = "select a.tbname,last(a.ts) cc from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts interval(1a) group by b.tbname limit %d" %(dbname,dbnamejoin,num) + self.sql_retun_error(sql_join,base_fun,replace_fun) + sql_join = "select last(*) from (%s)" %sql_join + self.sql_retun_error(sql_join,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_retun_error(sql_union,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_retun_error(sql_union_all,base_fun,replace_fun) + + + + tdLog.info("last query ---------13----------") + sql = "select tbname,last(ts) cc from %s.meters where ts is not null interval(1a) partition by tbname limit %d" %(dbname,num) + self.sql_retun_error(sql,base_fun,replace_fun) + sql = "select last(*) from (%s)" %sql + self.sql_retun_error(sql,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_retun_error(sql_union,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_retun_error(sql_union_all,base_fun,replace_fun) + + sql_join = "select a.tbname,last(a.ts) cc from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts interval(1a) partition by b.tbname limit %d" %(dbname,dbnamejoin,num) + self.sql_retun_error(sql_join,base_fun,replace_fun) + sql_join = "select last(*) from (%s)" %sql_join + self.sql_retun_error(sql_join,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_retun_error(sql_union,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_retun_error(sql_union_all,base_fun,replace_fun) + + + + tdLog.info("last query ---------14----------") + sql = "select tbname,last(*) cc from %s.meters where ts is not null partition by tbname interval(1a) limit %d" %(dbname,num) + self.sql_limit_retun_tables_times_n_slimit_return_per_table_num_times_n(sql,num,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_retun_tables_times_n_slimit_return_per_table_num_times_n(sql,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_retun_tables_times_n_slimit_return_per_table_num_times_n(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_retun_tables_times_n_slimit_return_per_table_num_times_n(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select a.tbname,last(a.ts) cc from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts partition by a.tbname interval(1a) limit %d" %(dbname,dbnamejoin,num) + self.sql_limit_retun_tables_times_n_slimit_return_per_table_num_times_n(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_retun_tables_times_n_slimit_return_per_table_num_times_n(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_retun_tables_times_n_slimit_return_per_table_num_times_n(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_retun_tables_times_n_slimit_return_per_table_num_times_n(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + + + tdLog.info("last query ---------15----------") + sql = "select tbname,last(ts) cc from %s.meters where ts is not null partition by tbname interval(1a) order by cc asc limit %d" %(dbname,num) + self.sql_limit_retun_n_slimit_return_per_table_num_times_tables(sql,num,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_retun_n_slimit_return_per_table_num_times_tables(sql,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_retun_n_slimit_return_per_table_num_times_tables(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_retun_n_slimit_return_per_table_num_times_tables(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select a.tbname,last(a.ts) cc from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts partition by a.tbname interval(1a) order by cc asc limit %d" %(dbname,dbnamejoin,num) + self.sql_limit_retun_n_slimit_return_per_table_num_times_tables(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_retun_n_slimit_return_per_table_num_times_tables(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_retun_n_slimit_return_per_table_num_times_tables(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_retun_n_slimit_return_per_table_num_times_tables(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + + + tdLog.info("last query ---------16----------") + sql = "select tbname,last(ts) cc from %s.meters where ts is not null partition by tbname interval(1a) order by cc desc limit %d" %(dbname,num) + self.sql_limit_retun_n_slimit_return_per_table_num_times_tables(sql,num,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_retun_n_slimit_return_per_table_num_times_tables(sql,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_retun_n_slimit_return_per_table_num_times_tables(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_retun_n_slimit_return_per_table_num_times_tables(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select a.tbname,last(a.ts) cc from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts partition by a.tbname interval(1a) order by cc desc limit %d" %(dbname,dbnamejoin,num) + self.sql_limit_retun_n_slimit_return_per_table_num_times_tables(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_retun_n_slimit_return_per_table_num_times_tables(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_retun_n_slimit_return_per_table_num_times_tables(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_retun_n_slimit_return_per_table_num_times_tables(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + + + tdLog.info("last query ---------17----------") + sql = "select tbname,last(*) cc from %s.meters where ts is not null partition by tbname interval(1a) slimit %d" %(dbname,num) + self.sql_limit_not_test_slimitkeep_return_per_table_num_times_n(sql,num,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_not_test_slimitkeep_return_per_table_num_times_n(sql,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_not_test_slimitkeep_return_per_table_num_times_n(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_not_test_slimitkeep_return_per_table_num_times_n(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select a.tbname,last(a.ts) cc from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts partition by a.tbname interval(1a) slimit %d" %(dbname,dbnamejoin,num) + self.sql_limit_not_test_slimitkeep_return_per_table_num_times_n(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_not_test_slimitkeep_return_per_table_num_times_n(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_not_test_slimitkeep_return_per_table_num_times_n(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_not_test_slimitkeep_return_per_table_num_times_n(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + + + tdLog.info("last query ---------18----------") + sql = "select tbname,last(ts) cc from %s.meters where ts is not null partition by tbname interval(1a) order by cc asc slimit %d" %(dbname,num) + self.sql_limit_not_test_slimitkeep_return_per_table_num_times_tables(sql,num,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_not_test_slimitkeep_return_per_table_num_times_tables(sql,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_not_test_slimitkeep_return_per_table_num_times_tables(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_not_test_slimitkeep_return_per_table_num_times_tables(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select a.tbname,last(a.ts) cc from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts partition by a.tbname interval(1a) order by cc asc slimit %d" %(dbname,dbnamejoin,num) + self.sql_limit_not_test_slimitkeep_return_per_table_num_times_tables(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_not_test_slimitkeep_return_per_table_num_times_tables(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_not_test_slimitkeep_return_per_table_num_times_tables(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_not_test_slimitkeep_return_per_table_num_times_tables(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + + tdLog.info("last query ---------19----------") + sql = "select tbname,last(ts) cc from %s.meters where ts is not null partition by tbname interval(1a) order by cc desc slimit %d" %(dbname,num) + self.sql_limit_not_test_slimitkeep_return_per_table_num_times_tables(sql,num,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_not_test_slimitkeep_return_per_table_num_times_tables(sql,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_not_test_slimitkeep_return_per_table_num_times_tables(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_not_test_slimitkeep_return_per_table_num_times_tables(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select a.tbname,last(a.ts) cc from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts partition by a.tbname interval(1a) order by cc desc slimit %d" %(dbname,dbnamejoin,num) + self.sql_limit_not_test_slimitkeep_return_per_table_num_times_tables(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_not_test_slimitkeep_return_per_table_num_times_tables(sql_join,num,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_not_test_slimitkeep_return_per_table_num_times_tables(sql_union,num,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_not_test_slimitkeep_return_per_table_num_times_tables(sql_union_all,num,tables,per_table_num,base_fun,replace_fun) + + + + tdLog.info("last query ---------20----------") + sql = "select tbname,last(*) cc from %s.meters where ts is not null partition by tbname interval(1a) slimit %d limit %d" %(dbname,num,num2) + self.sql_limit_retun_n_times_n2_slimitkeep_return_n_times_n2(sql,num,num2,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_retun_n_times_n2_slimitkeep_return_n_times_n2(sql,num,num2,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_retun_n_times_n2_slimitkeep_return_n_times_n2(sql_union,num,num2,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_retun_n_times_n2_slimitkeep_return_n_times_n2(sql_union_all,num,num2,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select a.tbname,last(a.*) cc from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts partition by a.tbname interval(1a) slimit %d limit %d" %(dbname,dbnamejoin,num,num2) + self.sql_limit_retun_n_times_n2_slimitkeep_return_n_times_n2(sql_join,num,num2,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_retun_n_times_n2_slimitkeep_return_n_times_n2(sql_join,num,num2,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_retun_n_times_n2_slimitkeep_return_n_times_n2(sql_union,num,num2,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_retun_n_times_n2_slimitkeep_return_n_times_n2(sql_union_all,num,num2,tables,per_table_num,base_fun,replace_fun) + + + tdLog.info("last query ---------21----------") + sql = "select tbname,last(ts) cc from %s.meters where ts is not null partition by tbname interval(1a) order by cc asc slimit %d limit %d" %(dbname,num,num2) + self.sql_limit_times_slimitkeep_return_n2(sql,num,num2,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_times_slimitkeep_return_n2(sql,num,num2,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_times_slimitkeep_return_n2(sql_union,num,num2,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_times_slimitkeep_return_n2(sql_union_all,num,num2,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select a.tbname,last(a.ts) cc from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts partition by a.tbname interval(1a) order by cc asc slimit %d limit %d" %(dbname,dbnamejoin,num,num2) + self.sql_limit_times_slimitkeep_return_n2(sql_join,num,num2,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_times_slimitkeep_return_n2(sql_join,num,num2,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_times_slimitkeep_return_n2(sql_union,num,num2,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_times_slimitkeep_return_n2(sql_union_all,num,num2,tables,per_table_num,base_fun,replace_fun) + + + tdLog.info("last query ---------22----------") + sql = "select tbname,last(ts) cc from %s.meters where ts is not null partition by tbname interval(1a) order by cc desc slimit %d limit %d" %(dbname,num,num2) + self.sql_limit_times_slimitkeep_return_n2(sql,num,num2,tables,per_table_num,base_fun,replace_fun) + sql = "select count(*) from (%s)" %sql + self.sql_data_limit_times_slimitkeep_return_n2(sql,num,num2,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql,sql) + self.sql_data_limit_times_slimitkeep_return_n2(sql_union,num,num2,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql,sql_union) + self.sql_data_limit_times_slimitkeep_return_n2(sql_union_all,num,num2,tables,per_table_num,base_fun,replace_fun) + + sql_join = "select a.tbname,last(a.ts) cc from %s.meters a,%s.meters b where a.ts is not null and a.ts = b.ts partition by a.tbname interval(1a) order by cc desc slimit %d limit %d" %(dbname,dbnamejoin,num,num2) + self.sql_limit_times_slimitkeep_return_n2(sql_join,num,num2,tables,per_table_num,base_fun,replace_fun) + sql_join = "select count(*) from (%s)" %sql_join + self.sql_data_limit_times_slimitkeep_return_n2(sql_join,num,num2,tables,per_table_num,base_fun,replace_fun) + sql_union = "(%s) union (%s)" %(sql_join,sql_join) + self.sql_data_limit_times_slimitkeep_return_n2(sql_union,num,num2,tables,per_table_num,base_fun,replace_fun) + sql_union_all = "(%s) union all (%s)" %(sql_join,sql_union) + self.sql_data_limit_times_slimitkeep_return_n2(sql_union_all,num,num2,tables,per_table_num,base_fun,replace_fun) + + + + + + def sql_base_check(self,sql1,sql2): + tdSql.query(sql1) + sql1_result = tdSql.getData(0,0) + tdLog.info("sql:%s , result: %s" %(sql1,sql1_result)) + + tdSql.query(sql2) + sql2_result = tdSql.getData(0,0) + tdLog.info("sql:%s , result: %s" %(sql2,sql2_result)) + + if sql1_result==sql2_result: + tdLog.info(f"checkEqual success, sql1_result={sql1_result},sql2_result={sql2_result}") + else : + tdLog.exit(f"checkEqual error, sql1_result=={sql1_result},sql2_result={sql2_result}") + + def run_limit_slimit_sql(self,dbname,tables,per_table_num,dbnamejoin): + + num,num2 = random.randint(10,100),random.randint(10,100) + self.sql_base(dbname,num,num2,tables,per_table_num,dbnamejoin) + + tdSql.execute(" flush database %s;" %dbname) + + self.sql_base(dbname,num,num2,tables,per_table_num,dbnamejoin) + + def check_sub(self,dbname): + + sql = "select count(*) from (select distinct(tbname) from %s.meters)" %dbname + self.sql_query_time_cost(sql) + num = tdSql.getData(0,0) + + for i in range(0,num): + sql1 = "select count(*) from %s.d%d" %(dbname,i) + self.sql_query_time_cost(sql1) + sql1_result = tdSql.getData(0,0) + tdLog.info("sql:%s , result: %s" %(sql1,sql1_result)) + + + def sql_base(self,dbname,num,num2,tables,per_table_num,dbnamejoin): + + sql = "select count(*) from %s.meters" %dbname + self.sql_query_time_cost(sql) + tdSql.checkData(0,0,tables*per_table_num) + sql = "select count(*) from %s.meters" %dbnamejoin + self.sql_query_time_cost(sql) + + self.fun_base(dbname,num,num2,tables,per_table_num,dbnamejoin,'*','*') + # self.fun_count(dbname,num,num2,tables,per_table_num,dbnamejoin,'count','count') + # self.fun_last(dbname,num,num2,tables,per_table_num,dbnamejoin,'last','last') + # #self.fun_last(dbname,num,num2,tables,per_table_num,dbnamejoin,'last','last_row') + # self.fun_last(dbname,num,num2,tables,per_table_num,dbnamejoin,'last','first') + + def test(self,dbname,tables,per_table_num,vgroups,replica,dbnamejoin): + self.run_benchmark(dbname,tables,per_table_num,vgroups,replica) + self.run_benchmark(dbnamejoin,tables,per_table_num,vgroups,replica) + self.run_limit_slimit_sql(dbname,tables,per_table_num,dbnamejoin) + + def run(self): + startTime = time.time() + + dbname = 'test' + dbnamejoin = 'testjoin' + vgroups = random.randint(1,8) + tables = random.randint(100,300) + per_table_num = random.randint(100,500) + replica = 1 + #self.test('test',tables,per_table_num,vgroup,1) + #self.test('test',10000,150,vgroup,1) + + self.test('test',100,150,vgroups,1,'testjoin') #方便调试，调试时不执行下面3个 + + # self.run_benchmark(dbname,tables,per_table_num,vgroups,replica) + # self.run_benchmark(dbnamejoin,tables*vgroups,per_table_num*vgroups,vgroups*2,replica) #方便测试不同数据量 + # self.run_limit_slimit_sql(dbname,tables,per_table_num,dbnamejoin) + + endTime = time.time() + print("total time %ds" % (endTime - startTime)) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 60a6f733fd60ec546f00422df4d716b719ceeb74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Mon, 24 Apr 2023 10:30:10 +0800 Subject: [PATCH 070/109] test: refine query cases --- tests/system-test/2-query/select_null.py | 446 +++++++++++++++++++++++ 1 file changed, 446 insertions(+) create mode 100755 tests/system-test/2-query/select_null.py diff --git a/tests/system-test/2-query/select_null.py b/tests/system-test/2-query/select_null.py new file mode 100755 index 0000000000..68eea8bc67 --- /dev/null +++ b/tests/system-test/2-query/select_null.py @@ -0,0 +1,446 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import random +import os +import time +import taos +import subprocess +from faker import Faker +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.dnodes import tdDnodes +from util.dnodes import * + +class TDTestCase: + + def init(self, conn, logSql, replicaVar): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.testcasePath = os.path.split(__file__)[0] + self.testcaseFilename = os.path.split(__file__)[-1] + os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename)) + + self.db = "sel_null" + + def insert_data(self,database,vgroups): + num_random = 10 + tdSql.execute('''drop database if exists %s ;''' %database) + tdSql.execute('''create database %s keep 36500 vgroups %d PRECISION 'us';'''%(database,vgroups)) + tdSql.execute('''use %s;'''%database) + + tdSql.execute('''create stable %s.stb0 (ts timestamp , c0 int , c1 double , c0null int , c1null double ) tags( t0 tinyint , t1 varchar(16) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint , t_bool bool , t_binary binary(100) , t_nchar nchar(100) ,t_float float , t_double double , t_ts timestamp);'''%database) + + for i in range(5): + tdSql.execute('''create table %s.stb0_%d using %s.stb0 tags(%d,'varchar%d',%d,%d, %d, %d,%d,'binary%d','nchar%d',%d,%d,%d ) ;'''%(database,i,database,i,i,i,i,i,i,i,i,i,i,i,i)) + + # insert data + for i in range(num_random): + for j in range(50): + tdSql.execute('''insert into %s.stb0_0 (ts , c1 , c0) values(now, %d, %d) ;''' % (database,j,j)) + tdSql.execute('''insert into %s.stb0_1 (ts , c1 , c0) values(now, %d, %d) ;''' % (database,j,j)) + tdSql.execute('''insert into %s.stb0_2 (ts , c1 , c0) values(now, %d, %d) ;''' % (database,j,j)) + tdSql.execute('''insert into %s.stb0_3 (ts , c1 , c0) values(now, %d, %d) ;''' % (database,j,j)) + tdSql.execute('''insert into %s.stb0_4 (ts , c1 , c0) values(now, %d, %d) ;''' % (database,j,j)) + + tdSql.query("select count(*) from %s.stb0;" %database) + tdSql.checkData(0,0,5*num_random*50) + tdSql.query("select count(*) from %s.stb0_0;"%database) + tdSql.checkData(0,0,num_random*50) + + def ts_3085(self,database): + sql = "select count(c0null) from(select * from %s.stb0 limit 20,4) "%(database) + tdSql.query(sql) + tdSql.checkData(0,0,0) + + offset = random.randint(10,100) + for i in range(offset): + sql = "select count(c0null) from(select * from %s.stb0 limit %d,%d) "%(database,offset,i) + tdSql.query(sql) + tdSql.checkData(0,0,0) + sql = "select count(c1null) from(select * from %s.stb0 limit %d,%d) "%(database,offset,i) + tdSql.query(sql) + tdSql.checkData(0,0,0) + sql = "select count(c0) from(select * from %s.stb0 limit %d,%d) "%(database,offset,i) + tdSql.query(sql) + tdSql.checkData(0,0,i) + sql = "select count(c1) from(select * from %s.stb0 limit %d,%d) "%(database,offset,i) + tdSql.query(sql) + tdSql.checkData(0,0,i) + sql = "select count(t0) from(select * from %s.stb0 limit %d,%d) "%(database,offset,i) + tdSql.query(sql) + tdSql.checkData(0,0,i) + sql = "select count(t1) from(select * from %s.stb0 limit %d,%d) "%(database,offset,i) + tdSql.query(sql) + tdSql.checkData(0,0,i) + + + def ts_2974_max(self,database): + sql = "select max(c0) from %s.stb0 where ts Date: Mon, 24 Apr 2023 10:31:27 +0800 Subject: [PATCH 071/109] test: refine query cases --- tests/parallel_test/cases.task | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 6e662a9a15..7bb8f08363 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -51,6 +51,16 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_26.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/select_null.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/select_null.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/select_null.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/select_null.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/select_null.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/create_wrong_topic.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3 From 3b8e8018b2991107b542c7c5830cbaae078d696b Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 24 Apr 2023 10:34:07 +0800 Subject: [PATCH 072/109] fix: get user errcode issue --- source/libs/catalog/src/ctgCache.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 0be2a78d48..ec087c1168 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -727,6 +727,7 @@ int32_t ctgGetCachedStbNameFromSuid(SCatalog* pCtg, char* dbFName, uint64_t suid } int32_t ctgChkAuthFromCache(SCatalog *pCtg, SUserAuthInfo *pReq, bool *inCache, SCtgAuthRsp *pRes) { + int32_t code = 0; if (IS_SYS_DBNAME(pReq->tbName.dbname)) { *inCache = true; pRes->pRawRes->pass = true; @@ -751,7 +752,7 @@ int32_t ctgChkAuthFromCache(SCatalog *pCtg, SUserAuthInfo *pReq, bool *inCache, CTG_LOCK(CTG_READ, &pUser->lock); memcpy(&req.authInfo, &pUser->userAuth, sizeof(pUser->userAuth)); - int32_t code = ctgChkSetAuthRes(pCtg, &req, pRes); + code = ctgChkSetAuthRes(pCtg, &req, pRes); CTG_UNLOCK(CTG_READ, &pUser->lock); CTG_ERR_JRET(code); From 905df6f7ac9b233b973d343b3e7990d79eef9b2a Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 24 Apr 2023 10:37:58 +0800 Subject: [PATCH 073/109] fix: memory leak issue --- source/libs/catalog/src/ctgUtil.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 62896e4307..4a5ddbac34 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -1371,6 +1371,8 @@ int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { tNameGetFullDbName(&req->pRawReq->tbName, dbFName); while (true) { + taosMemoryFreeClear(pMeta); + char* pCond = taosHashGet(pTbs, tbFName, strlen(tbFName)); if (pCond) { if (strlen(pCond) > 1) { @@ -1412,7 +1414,7 @@ int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { if (req->onlyCache) { res->metaNotExists = true; ctgDebug("suid %" PRIu64 " name not in cache for auth", pMeta->suid); - return TSDB_CODE_SUCCESS; + goto _return; } continue; From 8480c42234a2a948f296b5d318dd2125eeaafa5c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 24 Apr 2023 10:41:28 +0800 Subject: [PATCH 074/109] fix: memory leak issue --- source/libs/catalog/src/ctgUtil.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 4a5ddbac34..0ec7a92b84 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -1376,11 +1376,11 @@ int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { char* pCond = taosHashGet(pTbs, tbFName, strlen(tbFName)); if (pCond) { if (strlen(pCond) > 1) { - CTG_ERR_RET(nodesStringToNode(pCond, &res->pRawRes->pCond)); + CTG_ERR_JRET(nodesStringToNode(pCond, &res->pRawRes->pCond)); } res->pRawRes->pass = true; - return TSDB_CODE_SUCCESS; + goto _return; } if (stbName) { @@ -1388,19 +1388,19 @@ int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { goto _return; } - CTG_ERR_RET(catalogGetCachedTableMeta(pCtg, &req->pRawReq->tbName, &pMeta)); + CTG_ERR_JRET(catalogGetCachedTableMeta(pCtg, &req->pRawReq->tbName, &pMeta)); if (NULL == pMeta) { if (req->onlyCache) { res->metaNotExists = true; ctgDebug("db %s tb %s meta not in cache for auth", req->pRawReq->tbName.dbname, req->pRawReq->tbName.tname); - return TSDB_CODE_SUCCESS; + goto _return; } SCtgTbMetaCtx ctx = {0}; ctx.pName = (SName*)&req->pRawReq->tbName; ctx.flag = CTG_FLAG_UNKNOWN_STB | CTG_FLAG_SYNC_OP; - CTG_ERR_RET(ctgGetTbMeta(pCtg, req->pConn, &ctx, &pMeta)); + CTG_ERR_JRET(ctgGetTbMeta(pCtg, req->pConn, &ctx, &pMeta)); } if (TSDB_SUPER_TABLE == pMeta->tableType || TSDB_NORMAL_TABLE == pMeta->tableType) { @@ -1431,6 +1431,7 @@ int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { _return: taosMemoryFree(pMeta); + taosMemoryFree(stbName); CTG_RET(code); } From 24aff3d680d3cb2abd9d1fae2145bfcd2a935b7c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 24 Apr 2023 10:46:32 +0800 Subject: [PATCH 075/109] fix: memory leak issue --- source/libs/catalog/src/ctgUtil.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 0ec7a92b84..a42a189e75 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -528,11 +528,18 @@ void ctgFreeTaskRes(CTG_TASK_TYPE type, void** pRes) { } break; } + case CTG_TASK_GET_USER: { + if (*pRes) { + SUserAuthRes* pAuth = (SUserAuthRes *)*pRes; + nodesDestroyNode(pAuth->pCond); + taosMemoryFreeClear(*pRes); + } + break; + } case CTG_TASK_GET_TB_HASH: case CTG_TASK_GET_DB_INFO: case CTG_TASK_GET_INDEX_INFO: case CTG_TASK_GET_UDF: - case CTG_TASK_GET_USER: case CTG_TASK_GET_SVR_VER: case CTG_TASK_GET_TB_META: { taosMemoryFreeClear(*pRes); From 76949a4ba05e12d2040ce8f2d4975e369c97b832 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Mon, 24 Apr 2023 12:14:18 +0800 Subject: [PATCH 076/109] test: refine query cases --- tests/parallel_test/cases.task | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 7bb8f08363..c3841d3132 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -56,11 +56,11 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/select_null.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/select_null.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/select_null.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py.py -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py.py -R -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py.py -Q 4 +#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py.py +#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py.py -R +#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py.py -Q 2 +#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py.py -Q 3 +#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/create_wrong_topic.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3 From 1f6fc336c1678da694af9fd1ae698811ebdf77c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Mon, 24 Apr 2023 14:02:26 +0800 Subject: [PATCH 077/109] test: refine query cases --- tests/parallel_test/cases.task | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index c3841d3132..97858e5d71 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -56,11 +56,11 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/select_null.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/select_null.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/select_null.py -Q 4 -#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py.py -#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py.py -R -#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py.py -Q 2 -#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py.py -Q 3 -#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/create_wrong_topic.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3 From 5663550de22a2c88cb704d410b39c6aa1a0a0788 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 24 Apr 2023 15:47:02 +0800 Subject: [PATCH 078/109] enh(query): opt query perf by allocate the SLDataIter when opening reader. --- source/dnode/vnode/inc/vnode.h | 1 - source/dnode/vnode/src/inc/tsdb.h | 20 +++- source/dnode/vnode/src/tsdb/tsdbCache.c | 10 +- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 11 ++- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 101 ++++++-------------- source/dnode/vnode/src/tsdb/tsdbRead.c | 5 +- 6 files changed, 68 insertions(+), 80 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index fb2c2f4be3..aecfb9c3e5 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -162,7 +162,6 @@ int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType); #endif // tsdb -// typedef struct STsdb STsdb; typedef struct STsdbReader STsdbReader; #define TSDB_DEFAULT_STT_FILE 8 diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 2a85b191a4..8f6f9daf84 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -705,7 +705,6 @@ typedef struct SSttBlockLoadInfo { typedef struct SMergeTree { int8_t backward; SRBTree rbt; - SArray *pIterList; SLDataIter *pIter; bool destroyLoadInfo; SSttBlockLoadInfo *pLoadInfo; @@ -751,9 +750,25 @@ struct SDiskDataBuilder { SBlkInfo bi; }; +typedef struct SLDataIter { + SRBTreeNode node; + SSttBlk *pSttBlk; + SDataFReader *pReader; + int32_t iStt; + int8_t backward; + int32_t iSttBlk; + int32_t iRow; + SRowInfo rInfo; + uint64_t uid; + STimeWindow timeWindow; + SVersionRange verRange; + SSttBlockLoadInfo *pBlockLoadInfo; + bool ignoreEarlierTs; +} SLDataIter; + int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo, - bool destroyLoadInfo, const char *idStr, bool strictTimeRange); + bool destroyLoadInfo, const char *idStr, bool strictTimeRange, SLDataIter* pLDataIter); void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter); bool tMergeTreeNext(SMergeTree *pMTree); bool tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree); @@ -782,6 +797,7 @@ typedef struct SCacheRowsReader { STableKeyInfo *pTableList; // table id list int32_t numOfTables; SSttBlockLoadInfo *pLoadInfo; + SLDataIter *pDataIter; STsdbReadSnap *pReadSnap; SDataFReader *pDataFReader; SDataFReader *pDataFReaderLast; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 3c7edd931b..42f332c54a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -598,6 +598,7 @@ typedef struct { SMergeTree mergeTree; SMergeTree *pMergeTree; SSttBlockLoadInfo *pLoadInfo; + SLDataIter* pDataIter; int64_t lastTs; } SFSLastNextRowIter; @@ -645,7 +646,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa } tMergeTreeOpen(&state->mergeTree, 1, *state->pDataFReader, state->suid, state->uid, &(STimeWindow){.skey = state->lastTs, .ekey = TSKEY_MAX}, - &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, state->pLoadInfo, false, NULL, true); + &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, state->pLoadInfo, false, NULL, true, state->pDataIter); state->pMergeTree = &state->mergeTree; state->state = SFSLASTNEXTROW_BLOCKROW; } @@ -1211,7 +1212,7 @@ typedef struct { } CacheNextRowIter; static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTsdb, STSchema *pTSchema, tb_uid_t suid, - SSttBlockLoadInfo *pLoadInfo, STsdbReadSnap *pReadSnap, SDataFReader **pDataFReader, + SSttBlockLoadInfo *pLoadInfo, SLDataIter* pLDataIter, STsdbReadSnap *pReadSnap, SDataFReader **pDataFReader, SDataFReader **pDataFReaderLast, int64_t lastTs) { int code = 0; @@ -1274,6 +1275,7 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs pIter->fsLastState.pLoadInfo = pLoadInfo; pIter->fsLastState.pDataFReader = pDataFReaderLast; pIter->fsLastState.lastTs = lastTs; + pIter->fsLastState.pDataIter = pLDataIter; pIter->fsState.state = SFSNEXTROW_FS; pIter->fsState.pTsdb = pTsdb; @@ -1465,7 +1467,7 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, SArray **ppCo TSKEY lastRowTs = TSKEY_MAX; CacheNextRowIter iter = {0}; - nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->suid, pr->pLoadInfo, pr->pReadSnap, &pr->pDataFReader, + nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->suid, pr->pLoadInfo, pr->pDataIter, pr->pReadSnap, &pr->pDataFReader, &pr->pDataFReaderLast, pr->lastTs); do { @@ -1622,7 +1624,7 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach TSKEY lastRowTs = TSKEY_MAX; CacheNextRowIter iter = {0}; - nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->suid, pr->pLoadInfo, pr->pReadSnap, &pr->pDataFReader, + nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->suid, pr->pLoadInfo, pr->pDataIter, pr->pReadSnap, &pr->pDataFReader, &pr->pDataFReaderLast, pr->lastTs); do { diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index 95981c2f08..64d30c77a3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -187,13 +187,21 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, } } - int32_t numOfStt = ((SVnode*)pVnode)->config.sttTrigger; + SVnodeCfg* pCfg = &((SVnode*)pVnode)->config; + + int32_t numOfStt = pCfg->sttTrigger; p->pLoadInfo = tCreateLastBlockLoadInfo(p->pSchema, NULL, 0, numOfStt); if (p->pLoadInfo == NULL) { tsdbCacherowsReaderClose(p); return TSDB_CODE_OUT_OF_MEMORY; } + p->pDataIter = taosMemoryCalloc(pCfg->sttTrigger, sizeof(SLDataIter)); + if (p->pDataIter == NULL) { + tsdbCacherowsReaderClose(p); + return TSDB_CODE_OUT_OF_MEMORY; + } + p->idstr = taosStrdup(idstr); taosThreadMutexInit(&p->readerMutex, NULL); @@ -215,6 +223,7 @@ void* tsdbCacherowsReaderClose(void* pReader) { taosMemoryFree(p->pSchema); } + taosMemoryFreeClear(p->pDataIter); taosMemoryFree(p->pCurrSchema); destroyLastBlockLoadInfo(p->pLoadInfo); diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index eb383df48d..e354a50253 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -16,22 +16,6 @@ #include "tsdb.h" // SLDataIter ================================================= -struct SLDataIter { - SRBTreeNode node; - SSttBlk *pSttBlk; - SDataFReader *pReader; - int32_t iStt; - int8_t backward; - int32_t iSttBlk; - int32_t iRow; - SRowInfo rInfo; - uint64_t uid; - STimeWindow timeWindow; - SVersionRange verRange; - SSttBlockLoadInfo *pBlockLoadInfo; - bool ignoreEarlierTs; -}; - SSttBlockLoadInfo *tCreateLastBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols, int32_t numOfSttTrigger) { SSttBlockLoadInfo *pLoadInfo = taosMemoryCalloc(numOfSttTrigger, sizeof(SSttBlockLoadInfo)); @@ -268,25 +252,19 @@ static int32_t binarySearchForStartRowIndex(uint64_t *uidList, int32_t num, uint } } -int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t iStt, int8_t backward, uint64_t suid, +int32_t tLDataIterOpen(struct SLDataIter *pIter, SDataFReader *pReader, int32_t iStt, int8_t backward, uint64_t suid, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pRange, SSttBlockLoadInfo *pBlockLoadInfo, const char *idStr, bool strictTimeRange) { int32_t code = TSDB_CODE_SUCCESS; - *pIter = taosMemoryCalloc(1, sizeof(SLDataIter)); - if (*pIter == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } + pIter->uid = uid; + pIter->pReader = pReader; + pIter->iStt = iStt; + pIter->backward = backward; + pIter->verRange = *pRange; + pIter->timeWindow = *pTimeWindow; - (*pIter)->uid = uid; - (*pIter)->pReader = pReader; - (*pIter)->iStt = iStt; - (*pIter)->backward = backward; - (*pIter)->verRange = *pRange; - (*pIter)->timeWindow = *pTimeWindow; - - (*pIter)->pBlockLoadInfo = pBlockLoadInfo; + pIter->pBlockLoadInfo = pBlockLoadInfo; if (!pBlockLoadInfo->sttBlockLoaded) { int64_t st = taosGetTimestampUs(); @@ -294,7 +272,7 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t code = tsdbReadSttBlk(pReader, iStt, pBlockLoadInfo->aSttBlk); if (code) { - goto _exit; + return code; } // only apply to the child tables, ordinary tables will not incur this filter procedure. @@ -310,7 +288,7 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t // no qualified stt block existed taosArrayClear(pBlockLoadInfo->aSttBlk); - (*pIter)->iSttBlk = -1; + pIter->iSttBlk = -1; double el = (taosGetTimestampUs() - st) / 1000.0; tsdbDebug("load the last file info completed, elapsed time:%.2fms, %s", el, idStr); return code; @@ -343,31 +321,27 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); // find the start block - (*pIter)->iSttBlk = binarySearchForStartBlock(pBlockLoadInfo->aSttBlk->pData, size, uid, backward); - if ((*pIter)->iSttBlk != -1) { - (*pIter)->pSttBlk = taosArrayGet(pBlockLoadInfo->aSttBlk, (*pIter)->iSttBlk); - (*pIter)->iRow = ((*pIter)->backward) ? (*pIter)->pSttBlk->nRow : -1; + pIter->iSttBlk = binarySearchForStartBlock(pBlockLoadInfo->aSttBlk->pData, size, uid, backward); + if (pIter->iSttBlk != -1) { + pIter->pSttBlk = taosArrayGet(pBlockLoadInfo->aSttBlk, pIter->iSttBlk); + pIter->iRow = (pIter->backward) ? pIter->pSttBlk->nRow : -1; - if ((!backward) && ((strictTimeRange && (*pIter)->pSttBlk->minKey >= (*pIter)->timeWindow.ekey) || - (!strictTimeRange && (*pIter)->pSttBlk->minKey > (*pIter)->timeWindow.ekey))) { - (*pIter)->pSttBlk = NULL; + if ((!backward) && ((strictTimeRange && pIter->pSttBlk->minKey >= pIter->timeWindow.ekey) || + (!strictTimeRange && pIter->pSttBlk->minKey > pIter->timeWindow.ekey))) { + pIter->pSttBlk = NULL; } - if (backward && ((strictTimeRange && (*pIter)->pSttBlk->maxKey <= (*pIter)->timeWindow.skey) || - (!strictTimeRange && (*pIter)->pSttBlk->maxKey < (*pIter)->timeWindow.skey))) { - (*pIter)->pSttBlk = NULL; - (*pIter)->ignoreEarlierTs = true; + if (backward && ((strictTimeRange && pIter->pSttBlk->maxKey <= pIter->timeWindow.skey) || + (!strictTimeRange && pIter->pSttBlk->maxKey < pIter->timeWindow.skey))) { + pIter->pSttBlk = NULL; + pIter->ignoreEarlierTs = true; } } return code; - -_exit: - taosMemoryFree(*pIter); - return code; } -void tLDataIterClose(SLDataIter *pIter) { taosMemoryFree(pIter); } +void tLDataIterClose(SLDataIter *pIter) { /*taosMemoryFree(pIter); */} void tLDataIterNextBlock(SLDataIter *pIter, const char *idStr) { int32_t step = pIter->backward ? -1 : 1; @@ -594,43 +568,38 @@ static FORCE_INLINE int32_t tLDataIterDescCmprFn(const SRBTreeNode *p1, const SR int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo, - bool destroyLoadInfo, const char *idStr, bool strictTimeRange) { + bool destroyLoadInfo, const char *idStr, bool strictTimeRange, SLDataIter* pLDataIter) { + int32_t code = TSDB_CODE_SUCCESS; + pMTree->backward = backward; pMTree->pIter = NULL; - pMTree->pIterList = taosArrayInit(4, POINTER_BYTES); - if (pMTree->pIterList == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - pMTree->idStr = idStr; + if (!pMTree->backward) { // asc tRBTreeCreate(&pMTree->rbt, tLDataIterCmprFn); } else { // desc tRBTreeCreate(&pMTree->rbt, tLDataIterDescCmprFn); } - int32_t code = TSDB_CODE_SUCCESS; pMTree->pLoadInfo = pBlockLoadInfo; pMTree->destroyLoadInfo = destroyLoadInfo; pMTree->ignoreEarlierTs = false; for (int32_t i = 0; i < pFReader->pSet->nSttF; ++i) { // open all last file - struct SLDataIter *pIter = NULL; - code = tLDataIterOpen(&pIter, pFReader, i, pMTree->backward, suid, uid, pTimeWindow, pVerRange, + memset(&pLDataIter[i], 0, sizeof(SLDataIter)); + code = tLDataIterOpen(&pLDataIter[i], pFReader, i, pMTree->backward, suid, uid, pTimeWindow, pVerRange, &pMTree->pLoadInfo[i], pMTree->idStr, strictTimeRange); if (code != TSDB_CODE_SUCCESS) { goto _end; } - bool hasVal = tLDataIterNextRow(pIter, pMTree->idStr); + bool hasVal = tLDataIterNextRow(&pLDataIter[i], pMTree->idStr); if (hasVal) { - taosArrayPush(pMTree->pIterList, &pIter); - tMergeTreeAddIter(pMTree, pIter); + tMergeTreeAddIter(pMTree, &pLDataIter[i]); } else { if (!pMTree->ignoreEarlierTs) { - pMTree->ignoreEarlierTs = pIter->ignoreEarlierTs; + pMTree->ignoreEarlierTs = pLDataIter[i].ignoreEarlierTs; } - tLDataIterClose(pIter); } } @@ -681,15 +650,7 @@ bool tMergeTreeNext(SMergeTree *pMTree) { TSDBROW tMergeTreeGetRow(SMergeTree *pMTree) { return pMTree->pIter->rInfo.row; } void tMergeTreeClose(SMergeTree *pMTree) { - size_t size = taosArrayGetSize(pMTree->pIterList); - for (int32_t i = 0; i < size; ++i) { - SLDataIter *pIter = taosArrayGetP(pMTree->pIterList, i); - tLDataIterClose(pIter); - } - - pMTree->pIterList = taosArrayDestroy(pMTree->pIterList); pMTree->pIter = NULL; - if (pMTree->destroyLoadInfo) { pMTree->pLoadInfo = destroyLastBlockLoadInfo(pMTree->pLoadInfo); pMTree->destroyLoadInfo = false; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 5bd41dd86f..3aa3bdbd94 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -156,6 +156,7 @@ typedef struct SReaderStatus { SBlockData fileBlockData; SFilesetIter fileIter; SDataBlockIter blockIter; + SLDataIter* pLDataIter; } SReaderStatus; typedef struct SBlockInfoBuf { @@ -185,7 +186,6 @@ struct STsdbReader { STsdbReadSnap* pReadSnap; SIOCostSummary cost; STSchema* pSchema; // the newest version schema - // STSchema* pMemSchema; // the previous schema for in-memory data, to avoid load schema too many times SSHashObj* pSchemaMap; // keep the retrieved schema info, to avoid the overhead by repeatly load schema SDataFReader* pFileReader; // the file reader SDelFReader* pDelFReader; // the del file reader @@ -741,6 +741,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd pReader->type = pCond->type; pReader->window = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows); pReader->blockInfoBuf.numPerBucket = 1000; // 1000 tables per bucket + pReader->status.pLDataIter = taosMemoryCalloc(pVnode->config.sttTrigger, sizeof(SLDataIter)); if (pReader->pResBlock == NULL) { pReader->freeBlock = true; @@ -2547,7 +2548,7 @@ static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScan pScanInfo->uid, pReader->idStr); int32_t code = tMergeTreeOpen(&pLBlockReader->mergeTree, (pLBlockReader->order == TSDB_ORDER_DESC), pReader->pFileReader, pReader->suid, pScanInfo->uid, &w, &pLBlockReader->verRange, - pLBlockReader->pInfo, false, pReader->idStr, false); + pLBlockReader->pInfo, false, pReader->idStr, false, pReader->status.pLDataIter); if (code != TSDB_CODE_SUCCESS) { return false; } From d73306f6165a1cfec3091625096aa89c9763193e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 24 Apr 2023 16:08:49 +0800 Subject: [PATCH 079/109] enh(query): opt last row read performance. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 52 ++++++++++++++------------ 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 3aa3bdbd94..b40e5abf8e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -741,7 +741,6 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd pReader->type = pCond->type; pReader->window = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows); pReader->blockInfoBuf.numPerBucket = 1000; // 1000 tables per bucket - pReader->status.pLDataIter = taosMemoryCalloc(pVnode->config.sttTrigger, sizeof(SLDataIter)); if (pReader->pResBlock == NULL) { pReader->freeBlock = true; @@ -774,6 +773,12 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd goto _end; } + pReader->status.pLDataIter = taosMemoryCalloc(pVnode->config.sttTrigger, sizeof(SLDataIter)); + if (pReader->status.pLDataIter == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + setColumnIdSlotList(pSup, pCond->colList, pCond->pSlotList, pCond->numOfCols); tsdbInitReaderLock(pReader); @@ -4368,29 +4373,38 @@ _err: return code; } +static void clearSharedPtr(STsdbReader* p) { + p->status.pLDataIter = NULL; + p->status.pTableMap = NULL; + p->status.uidList.tableUidList = NULL; + p->pReadSnap = NULL; + p->pSchema = NULL; + p->pSchemaMap = NULL; +} + +static void setSharedPtr(STsdbReader* pDst, const STsdbReader* pSrc) { + pDst->status.pTableMap = pSrc->status.pTableMap; + pDst->status.pLDataIter = pSrc->status.pLDataIter; + pDst->status.uidList = pSrc->status.uidList; + pDst->pSchema = pSrc->pSchema; + pDst->pSchemaMap = pSrc->pSchemaMap; + pDst->pReadSnap = pSrc->pReadSnap; +} + void tsdbReaderClose(STsdbReader* pReader) { if (pReader == NULL) { return; } tsdbAcquireReader(pReader); + { if (pReader->innerReader[0] != NULL || pReader->innerReader[1] != NULL) { STsdbReader* p = pReader->innerReader[0]; - - p->status.pTableMap = NULL; - p->status.uidList.tableUidList = NULL; - p->pReadSnap = NULL; - p->pSchema = NULL; - p->pSchemaMap = NULL; + clearSharedPtr(p); p = pReader->innerReader[1]; - - p->status.pTableMap = NULL; - p->status.uidList.tableUidList = NULL; - p->pReadSnap = NULL; - p->pSchema = NULL; - p->pSchemaMap = NULL; + clearSharedPtr(p); tsdbReaderClose(pReader->innerReader[0]); tsdbReaderClose(pReader->innerReader[1]); @@ -4627,18 +4641,10 @@ int32_t tsdbReaderResume(STsdbReader* pReader) { // we need only one row pPrevReader->capacity = 1; - pPrevReader->status.pTableMap = pReader->status.pTableMap; - pPrevReader->status.uidList = pReader->status.uidList; - pPrevReader->pSchema = pReader->pSchema; - pPrevReader->pSchemaMap = pReader->pSchemaMap; - pPrevReader->pReadSnap = pReader->pReadSnap; + setSharedPtr(pPrevReader, pReader); pNextReader->capacity = 1; - pNextReader->status.pTableMap = pReader->status.pTableMap; - pNextReader->status.uidList = pReader->status.uidList; - pNextReader->pSchema = pReader->pSchema; - pNextReader->pSchemaMap = pReader->pSchemaMap; - pNextReader->pReadSnap = pReader->pReadSnap; + setSharedPtr(pNextReader, pReader); code = doOpenReaderImpl(pPrevReader); if (code != TSDB_CODE_SUCCESS) { From 7360a6b65ca75f8ebc0e48f3d641ef6341503899 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 24 Apr 2023 16:39:55 +0800 Subject: [PATCH 080/109] enh(query): opt hash. --- source/dnode/vnode/src/inc/tsdb.h | 3 +- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 5 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 124 +++++++++++---------- source/util/src/tsimplehash.c | 4 - 4 files changed, 69 insertions(+), 67 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 8f6f9daf84..c3a0db466b 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -16,6 +16,7 @@ #ifndef _TD_VNODE_TSDB_H_ #define _TD_VNODE_TSDB_H_ +#include "tsimplehash.h" #include "vnodeInt.h" #ifdef __cplusplus @@ -224,7 +225,7 @@ int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, void *tsdbTbDataIterDestroy(STbDataIter *pIter); void tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter *pIter); bool tsdbTbDataIterNext(STbDataIter *pIter); -void tsdbMemTableCountRows(SMemTable *pMemTable, SHashObj *pTableMap, int64_t *rowsNum); +void tsdbMemTableCountRows(SMemTable *pMemTable, SSHashObj *pTableMap, int64_t *rowsNum); // STbData int32_t tsdbGetNRowsInTbData(STbData *pTbData); diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index d0ff403bf7..f27a28acb3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include #include "tsdb.h" #define MEM_MIN_HASH 1024 @@ -298,12 +299,12 @@ int64_t tsdbCountTbDataRows(STbData *pTbData) { return rowsNum; } -void tsdbMemTableCountRows(SMemTable *pMemTable, SHashObj* pTableMap, int64_t *rowsNum) { +void tsdbMemTableCountRows(SMemTable *pMemTable, SSHashObj* pTableMap, int64_t *rowsNum) { taosRLockLatch(&pMemTable->latch); for (int32_t i = 0; i < pMemTable->nBucket; ++i) { STbData *pTbData = pMemTable->aBucket[i]; while (pTbData) { - void* p = taosHashGet(pTableMap, &pTbData->uid, sizeof(pTbData->uid)); + void* p = tSimpleHashGet(pTableMap, &pTbData->uid, sizeof(pTbData->uid)); if (p == NULL) { pTbData = pTbData->next; continue; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index b40e5abf8e..042b31ee09 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -125,12 +125,12 @@ typedef struct SFileDataBlockInfo { } SFileDataBlockInfo; typedef struct SDataBlockIter { - int32_t numOfBlocks; - int32_t index; - SArray* blockList; // SArray - int32_t order; - SDataBlk block; // current SDataBlk data - SHashObj* pTableMap; + int32_t numOfBlocks; + int32_t index; + SArray* blockList; // SArray + int32_t order; + SDataBlk block; // current SDataBlk data + SSHashObj* pTableMap; } SDataBlockIter; typedef struct SFileBlockDumpInfo { @@ -148,7 +148,7 @@ typedef struct STableUidList { typedef struct SReaderStatus { bool loadFromFile; // check file stage bool composedDataBlock; // the returned data block is a composed block or not - SHashObj* pTableMap; // SHash + SSHashObj* pTableMap; // SHash STableBlockScanInfo** pTableIter; // table iterator used in building in-memory buffer data blocks. STableUidList uidList; // check tables in uid order, to avoid the repeatly load of blocks in STT. SFileBlockDumpInfo fBlockDumpInfo; @@ -233,7 +233,7 @@ static bool hasDataInFileBlock(const SBlockData* pBlockData, const SFil static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter); static int32_t getInitialDelIndex(const SArray* pDelSkyline, int32_t order); -static STableBlockScanInfo* getTableBlockScanInfo(SHashObj* pTableMap, uint64_t uid, const char* id); +static STableBlockScanInfo* getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, const char* id); static FORCE_INLINE STSchema* getLatestTableSchema(STsdbReader* pReader, uint64_t uid); @@ -384,12 +384,11 @@ static int32_t uidComparFunc(const void* p1, const void* p2) { } // NOTE: speedup the whole processing by preparing the buffer for STableBlockScanInfo in batch model -static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList, +static SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList, STableUidList* pUidList, int32_t numOfTables) { // allocate buffer in order to load data blocks from file // todo use simple hash instead, optimize the memory consumption - SHashObj* pTableMap = - taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + SSHashObj* pTableMap = tSimpleHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); if (pTableMap == NULL) { return NULL; } @@ -399,7 +398,7 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf pUidList->tableUidList = taosMemoryMalloc(numOfTables * sizeof(uint64_t)); if (pUidList->tableUidList == NULL) { - taosHashCleanup(pTableMap); + tSimpleHashCleanup(pTableMap); return NULL; } @@ -421,7 +420,7 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf pScanInfo->lastKeyInStt = ekey; } - taosHashPut(pTableMap, &pScanInfo->uid, sizeof(uint64_t), &pScanInfo, POINTER_BYTES); + tSimpleHashPut(pTableMap, &pScanInfo->uid, sizeof(uint64_t), &pScanInfo, POINTER_BYTES); tsdbTrace("%p check table uid:%" PRId64 " from lastKey:%" PRId64 " %s", pTsdbReader, pScanInfo->uid, pScanInfo->lastKey, pTsdbReader->idStr); } @@ -436,9 +435,11 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf return pTableMap; } -static void resetAllDataBlockScanInfo(SHashObj* pTableMap, int64_t ts, int32_t step) { - STableBlockScanInfo** p = NULL; - while ((p = taosHashIterate(pTableMap, p)) != NULL) { +static void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step) { + void *p = NULL; + int32_t iter = 0; + + while ((p = tSimpleHashIterate(pTableMap, p, &iter)) != NULL) { STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; pInfo->iterInit = false; @@ -478,13 +479,15 @@ static void clearBlockScanInfo(STableBlockScanInfo* p) { tMapDataClear(&p->mapData); } -static void destroyAllBlockScanInfo(SHashObj* pTableMap) { +static void destroyAllBlockScanInfo(SSHashObj* pTableMap) { void* p = NULL; - while ((p = taosHashIterate(pTableMap, p)) != NULL) { + int32_t iter = 0; + + while ((p = tSimpleHashIterate(pTableMap, p, &iter)) != NULL) { clearBlockScanInfo(*(STableBlockScanInfo**)p); } - taosHashCleanup(pTableMap); + tSimpleHashCleanup(pTableMap); } static bool isEmptyQueryTimeWindow(STimeWindow* pWindow) { return pWindow->skey > pWindow->ekey; } @@ -800,7 +803,7 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFReader* pFileReader, goto _end; } - int32_t numOfTables = taosHashGetSize(pReader->status.pTableMap); + int32_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); SArray* aBlockIdx = (SArray*)taosLRUCacheValue(pFileReader->pTsdb->biCache, handle); size_t num = taosArrayGetSize(aBlockIdx); @@ -864,10 +867,12 @@ _end: return code; } -static void cleanupTableScanInfo(SHashObj* pTableMap) { +static void cleanupTableScanInfo(SSHashObj* pTableMap) { STableBlockScanInfo** px = NULL; + int32_t iter = 0; + while (1) { - px = taosHashIterate(pTableMap, px); + px = tSimpleHashIterate(pTableMap, px, &iter); if (px == NULL) { break; } @@ -1439,7 +1444,7 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte pBlockIter->pTableMap = pReader->status.pTableMap; // access data blocks according to the offset of each block in asc/desc order. - int32_t numOfTables = (int32_t)taosHashGetSize(pReader->status.pTableMap); + int32_t numOfTables = (int32_t)tSimpleHashGetSize(pReader->status.pTableMap); int64_t st = taosGetTimestampUs(); int32_t code = initBlockOrderSupporter(&sup, numOfTables); @@ -1449,8 +1454,10 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte int32_t cnt = 0; void* ptr = NULL; + int32_t iter = 0; + while (1) { - ptr = taosHashIterate(pReader->status.pTableMap, ptr); + ptr = tSimpleHashIterate(pReader->status.pTableMap, ptr, &iter); if (ptr == NULL) { break; } @@ -2916,7 +2923,7 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { pBlockNum->numOfBlocks = 0; pBlockNum->numOfLastFiles = 0; - size_t numOfTables = taosHashGetSize(pReader->status.pTableMap); + size_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); SArray* pIndexList = taosArrayInit(numOfTables, sizeof(SBlockIdx)); while (1) { @@ -2985,18 +2992,18 @@ static void resetTableListIndex(SReaderStatus* pStatus) { pList->currentIndex = 0; uint64_t uid = pList->tableUidList[0]; - pStatus->pTableIter = taosHashGet(pStatus->pTableMap, &uid, sizeof(uid)); + pStatus->pTableIter = tSimpleHashGet(pStatus->pTableMap, &uid, sizeof(uid)); } static bool moveToNextTable(STableUidList* pOrderedCheckInfo, SReaderStatus* pStatus) { pOrderedCheckInfo->currentIndex += 1; - if (pOrderedCheckInfo->currentIndex >= taosHashGetSize(pStatus->pTableMap)) { + if (pOrderedCheckInfo->currentIndex >= tSimpleHashGetSize(pStatus->pTableMap)) { pStatus->pTableIter = NULL; return false; } uint64_t uid = pOrderedCheckInfo->tableUidList[pOrderedCheckInfo->currentIndex]; - pStatus->pTableIter = taosHashGet(pStatus->pTableMap, &uid, sizeof(uid)); + pStatus->pTableIter = tSimpleHashGet(pStatus->pTableMap, &uid, sizeof(uid)); return (pStatus->pTableIter != NULL); } @@ -3006,7 +3013,7 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { STableUidList* pUidList = &pStatus->uidList; int32_t code = TSDB_CODE_SUCCESS; - if (taosHashGetSize(pStatus->pTableMap) == 0) { + if (tSimpleHashGetSize(pStatus->pTableMap) == 0) { return TSDB_CODE_SUCCESS; } @@ -3168,7 +3175,7 @@ static int32_t doSumFileBlockRows(STsdbReader* pReader, SDataFReader* pFileReade goto _end; } - int32_t numOfTables = taosHashGetSize(pReader->status.pTableMap); + int32_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); SArray* aBlockIdx = (SArray*)taosLRUCacheValue(pFileReader->pTsdb->biCache, handle); size_t num = taosArrayGetSize(aBlockIdx); @@ -3178,14 +3185,13 @@ static int32_t doSumFileBlockRows(STsdbReader* pReader, SDataFReader* pFileReade } SBlockIdx* pBlockIdx = NULL; - int32_t i = 0; for (int32_t i = 0; i < num; ++i) { pBlockIdx = (SBlockIdx*)taosArrayGet(aBlockIdx, i); if (pBlockIdx->suid != pReader->suid) { continue; } - STableBlockScanInfo** p = taosHashGet(pReader->status.pTableMap, &pBlockIdx->uid, sizeof(pBlockIdx->uid)); + STableBlockScanInfo** p = tSimpleHashGet(pReader->status.pTableMap, &pBlockIdx->uid, sizeof(pBlockIdx->uid)); if (p == NULL) { continue; } @@ -3231,13 +3237,13 @@ static int32_t doSumSttBlockRows(STsdbReader* pReader) { taosArrayClear(pBlockLoadInfo->aSttBlk); continue; } - for (int32_t i = 0; i < size; ++i) { - SSttBlk* p = taosArrayGet(pBlockLoadInfo->aSttBlk, i); + for (int32_t j = 0; j < size; ++j) { + SSttBlk* p = taosArrayGet(pBlockLoadInfo->aSttBlk, j); pReader->rowsNum += p->nRow; } } else { - for (int32_t i = 0; i < size; ++i) { - SSttBlk* p = taosArrayGet(pBlockLoadInfo->aSttBlk, i); + for (int32_t j = 0; j < size; ++j) { + SSttBlk* p = taosArrayGet(pBlockLoadInfo->aSttBlk, j); uint64_t s = p->suid; if (s < pReader->suid) { continue; @@ -3307,13 +3313,6 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) { STableUidList* pUidList = &pStatus->uidList; while (1) { - // if (pStatus->pTableIter == NULL) { - // pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, NULL); - // if (pStatus->pTableIter == NULL) { - // return TSDB_CODE_SUCCESS; - // } - // } - STableBlockScanInfo** pBlockScanInfo = pStatus->pTableIter; initMemDataIterator(*pBlockScanInfo, pReader); @@ -3341,7 +3340,7 @@ static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) SDataBlk* pBlock = getCurrentBlock(pBlockIter); SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); if (pBlockInfo) { - STableBlockScanInfo* pScanInfo = taosHashGet(pBlockIter->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); + STableBlockScanInfo* pScanInfo = tSimpleHashGet(pBlockIter->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); if (pScanInfo) { lastKey = pScanInfo->lastKey; } @@ -4169,10 +4168,12 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e // TODO refactor: with createDataBlockScanInfo int32_t tsdbSetTableList(STsdbReader* pReader, const void* pTableList, int32_t num) { - int32_t size = taosHashGetSize(pReader->status.pTableMap); + int32_t size = tSimpleHashGetSize(pReader->status.pTableMap); STableBlockScanInfo** p = NULL; - while ((p = taosHashIterate(pReader->status.pTableMap, p)) != NULL) { + int32_t iter = 0; + + while ((p = tSimpleHashIterate(pReader->status.pTableMap, p, &iter)) != NULL) { clearBlockScanInfo(*p); } @@ -4190,7 +4191,7 @@ int32_t tsdbSetTableList(STsdbReader* pReader, const void* pTableList, int32_t n pReader->status.uidList.tableUidList = (uint64_t*)p1; } - taosHashClear(pReader->status.pTableMap); + tSimpleHashClear(pReader->status.pTableMap); STableUidList* pUidList = &pReader->status.uidList; pUidList->currentIndex = 0; @@ -4211,7 +4212,7 @@ int32_t tsdbSetTableList(STsdbReader* pReader, const void* pTableList, int32_t n pInfo->lastKeyInStt = ekey; } - taosHashPut(pReader->status.pTableMap, &pInfo->uid, sizeof(uint64_t), &pInfo, POINTER_BYTES); + tSimpleHashPut(pReader->status.pTableMap, &pInfo->uid, sizeof(uint64_t), &pInfo, POINTER_BYTES); } return TDB_CODE_SUCCESS; @@ -4428,7 +4429,7 @@ void tsdbReaderClose(STsdbReader* pReader) { tBlockDataDestroy(&pReader->status.fileBlockData); cleanupDataBlockIterator(&pReader->status.blockIter); - size_t numOfTables = taosHashGetSize(pReader->status.pTableMap); + size_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); if (pReader->status.pTableMap != NULL) { destroyAllBlockScanInfo(pReader->status.pTableMap); clearBlockScanInfoBuf(&pReader->blockInfoBuf); @@ -4510,8 +4511,9 @@ int32_t tsdbReaderSuspend(STsdbReader* pReader) { // resetDataBlockScanInfo excluding lastKey STableBlockScanInfo** p = NULL; + int32_t iter = 0; - while ((p = taosHashIterate(pStatus->pTableMap, p)) != NULL) { + while ((p = tSimpleHashIterate(pStatus->pTableMap, p, &iter)) != NULL) { STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; pInfo->iterInit = false; @@ -4532,8 +4534,9 @@ int32_t tsdbReaderSuspend(STsdbReader* pReader) { } else { // resetDataBlockScanInfo excluding lastKey STableBlockScanInfo** p = NULL; + int32_t iter = 0; - while ((p = taosHashIterate(pStatus->pTableMap, p)) != NULL) { + while ((p = tSimpleHashIterate(pStatus->pTableMap, p, &iter)) != NULL) { STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; pInfo->iterInit = false; @@ -4622,7 +4625,7 @@ int32_t tsdbReaderResume(STsdbReader* pReader) { // restore reader's state // task snapshot - int32_t numOfTables = taosHashGetSize(pReader->status.pTableMap); + int32_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); if (numOfTables > 0) { qTrace("tsdb/reader: %p, take snapshot", pReader); code = tsdbTakeReadSnap(pReader, tsdbSetQueryReseek, &pReader->pReadSnap); @@ -4701,7 +4704,7 @@ static int32_t doTsdbNextDataBlock(STsdbReader* pReader, bool* hasNext) { *hasNext = false; SReaderStatus* pStatus = &pReader->status; - if (taosHashGetSize(pStatus->pTableMap) == 0) { + if (tSimpleHashGetSize(pStatus->pTableMap) == 0) { return code; } @@ -4954,11 +4957,11 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, return code; } -STableBlockScanInfo* getTableBlockScanInfo(SHashObj* pTableMap, uint64_t uid, const char* id) { - STableBlockScanInfo** p = taosHashGet(pTableMap, &uid, sizeof(uid)); +STableBlockScanInfo* getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, const char* id) { + STableBlockScanInfo** p = tSimpleHashGet(pTableMap, &uid, sizeof(uid)); if (p == NULL || *p == NULL) { terrno = TSDB_CODE_INVALID_PARA; - int32_t size = taosHashGetSize(pTableMap); + int32_t size = tSimpleHashGetSize(pTableMap); tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", uid, size, id); return NULL; } @@ -5044,7 +5047,7 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { pReader->suppInfo.tsColAgg.colId = PRIMARYKEY_TIMESTAMP_COL_ID; tsdbDataFReaderClose(&pReader->pFileReader); - int32_t numOfTables = taosHashGetSize(pStatus->pTableMap); + int32_t numOfTables = tSimpleHashGetSize(pStatus->pTableMap); initFilesetIterator(&pStatus->fileIter, pReader->pReadSnap->fs.aDFileSet, pReader); resetDataBlockIterator(pBlockIter, pReader->order); @@ -5115,7 +5118,7 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa pTableBlockInfo->numOfFiles += 1; - int32_t numOfTables = (int32_t)taosHashGetSize(pStatus->pTableMap); + int32_t numOfTables = (int32_t)tSimpleHashGetSize(pStatus->pTableMap); int defaultRows = 4096; SDataBlockIter* pBlockIter = &pStatus->blockIter; @@ -5179,7 +5182,8 @@ int64_t tsdbGetNumOfRowsInMemTable(STsdbReader* pReader) { tsdbReaderResume(pReader); } - pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, NULL); + int32_t iter = 0; + pStatus->pTableIter = tSimpleHashIterate(pStatus->pTableMap, NULL, &iter); while (pStatus->pTableIter != NULL) { STableBlockScanInfo* pBlockScanInfo = *(STableBlockScanInfo**)pStatus->pTableIter; @@ -5201,7 +5205,7 @@ int64_t tsdbGetNumOfRowsInMemTable(STsdbReader* pReader) { } // current table is exhausted, let's try the next table - pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, pStatus->pTableIter); + pStatus->pTableIter = tSimpleHashIterate(pStatus->pTableMap, pStatus->pTableIter, &iter); } tsdbReleaseReader(pReader); diff --git a/source/util/src/tsimplehash.c b/source/util/src/tsimplehash.c index ec1991923f..4c7983a983 100644 --- a/source/util/src/tsimplehash.c +++ b/source/util/src/tsimplehash.c @@ -361,10 +361,6 @@ int32_t tSimpleHashIterateRemove(SSHashObj *pHashObj, const void *key, size_t ke return TSDB_CODE_SUCCESS; } -static void destroyItems(void* pItem) { - taosMemoryFree(*(void**)pItem); -} - void tSimpleHashClear(SSHashObj *pHashObj) { if (!pHashObj || taosHashTableEmpty(pHashObj)) { return; From 8c4b73a351b3f5e8c05b9babe9fd42298abc865a Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 24 Apr 2023 19:19:12 +0800 Subject: [PATCH 081/109] feat: add get table tag sync API --- include/libs/catalog/catalog.h | 2 ++ source/libs/catalog/src/catalog.c | 57 +++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 6f2fb4eb6b..429e7ffa73 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -314,6 +314,8 @@ int32_t catalogGetIndexMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const char* int32_t catalogGetTableIndex(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pRes); +int32_t catalogGetTableTag(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pRes); + int32_t catalogRefreshGetTableCfg(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, STableCfg** pCfg); int32_t catalogUpdateTableIndex(SCatalog* pCtg, STableIndexRsp* pRsp); diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index b263654e70..976a38c03d 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -429,6 +429,48 @@ int32_t ctgGetTbCfg(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, CTG_RET(TSDB_CODE_SUCCESS); } +int32_t ctgGetTbTag(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, SArray** pRes) { + SVgroupInfo vgroupInfo = {0}; + STableCfg* pCfg = NULL; + int32_t code = 0; + + CTG_ERR_RET(ctgGetTbHashVgroup(pCtg, pConn, pTableName, &vgroupInfo, NULL)); + CTG_ERR_RET(ctgGetTableCfgFromVnode(pCtg, pConn, pTableName, &vgroupInfo, &pCfg, NULL)); + + if (NULL == pCfg->pTags || pCfg->tagsLen <= 0) { + ctgError("invalid tag in tbCfg rsp, pTags:%p, len:%d", pCfg->pTags, pCfg->tagsLen); + CTG_ERR_JRET(TSDB_CODE_INVALID_MSG); + } + + SArray* pTagVals = NULL; + STag* pTag = (STag*)pCfg->pTags; + + if (tTagIsJson(pTag)) { + pTagVals = taosArrayInit(1, sizeof(STagVal)); + if (NULL == pTagVals) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + char* pJson = parseTagDatatoJson(pTag); + STagVal tagVal; + tagVal.cid = 0; + tagVal.type = TSDB_DATA_TYPE_JSON; + tagVal.pData = pJson; + tagVal.nData = strlen(pJson); + taosArrayPush(pTagVals, &tagVal); + } else { + CTG_ERR_JRET(tTagToValArray((const STag*)pCfg->pTags, &pTagVals)); + } + + *pRes = pTagVals; + +_return: + + tFreeSTableCfgRsp((STableCfgRsp*)pCfg); + + CTG_RET(code); +} + int32_t ctgGetTbDistVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, SArray** pVgList) { STableMeta* tbMeta = NULL; int32_t code = 0; @@ -1414,6 +1456,21 @@ _return: CTG_API_LEAVE(code); } +int32_t catalogGetTableTag(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pRes) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == pConn || NULL == pTableName || NULL == pRes) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + int32_t code = 0; + CTG_ERR_JRET(ctgGetTbTag(pCtg, pConn, (SName*)pTableName, pRes)); + +_return: + + CTG_API_LEAVE(code); +} + int32_t catalogRefreshGetTableCfg(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, STableCfg** pCfg) { CTG_API_ENTER(); From 59ae3ecc2c17130049beace271c5950875bb9800 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 24 Apr 2023 20:10:39 +0800 Subject: [PATCH 082/109] opti:test cases for tmq --- tests/system-test/7-tmq/subscribeDb3.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/system-test/7-tmq/subscribeDb3.py b/tests/system-test/7-tmq/subscribeDb3.py index 22004a95f1..5b5326cfba 100644 --- a/tests/system-test/7-tmq/subscribeDb3.py +++ b/tests/system-test/7-tmq/subscribeDb3.py @@ -10,6 +10,7 @@ from util.log import * from util.sql import * from util.cases import * from util.dnodes import * +sys.path.append("./7-tmq") from tmqCommon import * class TDTestCase: From c6daa68e5420f893218503c0000cb971ea1d9438 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Apr 2023 00:17:37 +0800 Subject: [PATCH 083/109] enh(query): opt merge life cycle. --- source/dnode/vnode/src/inc/tsdb.h | 8 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 189 +++++++++++++------------ source/dnode/vnode/src/tsdb/tsdbUtil.c | 78 ++++------ 3 files changed, 134 insertions(+), 141 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index c3a0db466b..b2b04abff1 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -126,11 +126,13 @@ SColVal *tsdbRowIterNext(STSDBRowIter *pIter); // SRowMerger int32_t tsdbRowMergerInit(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema); int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); - -// int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); void tsdbRowMergerClear(SRowMerger *pMerger); -// int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow); int32_t tsdbRowMergerGetRow(SRowMerger *pMerger, SRow **ppRow); + +int32_t tsdbRowMergerInit_rv(SRowMerger* pMerger, STSchema *pSchema); +void tsdbRowMergerClear_rv(SRowMerger* pMerger); +void tsdbRowMergerCleanup_rv(SRowMerger* pMerger); + // TABLEID int32_t tTABLEIDCmprFn(const void *p1, const void *p2); // TSDBKEY diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 042b31ee09..dd6913039c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -157,6 +157,7 @@ typedef struct SReaderStatus { SFilesetIter fileIter; SDataBlockIter blockIter; SLDataIter* pLDataIter; + SRowMerger merger; } SReaderStatus; typedef struct SBlockInfoBuf { @@ -166,6 +167,15 @@ typedef struct SBlockInfoBuf { int32_t numOfTables; } SBlockInfoBuf; +typedef struct STsdbReaderAttr { + STSchema* pSchema; + EReadMode readMode; + uint64_t rowsNum; + STimeWindow window; + bool freeBlock; + SVersionRange verRange; +} STsdbReaderAttr; + struct STsdbReader { STsdb* pTsdb; SVersionRange verRange; @@ -185,28 +195,26 @@ struct STsdbReader { SBlockLoadSuppInfo suppInfo; STsdbReadSnap* pReadSnap; SIOCostSummary cost; - STSchema* pSchema; // the newest version schema - SSHashObj* pSchemaMap; // keep the retrieved schema info, to avoid the overhead by repeatly load schema - SDataFReader* pFileReader; // the file reader - SDelFReader* pDelFReader; // the del file reader - SArray* pDelIdx; // del file block index; - SBlockInfoBuf blockInfoBuf; - int32_t step; - STsdbReader* innerReader[2]; + STSchema* pSchema; // the newest version schema + SSHashObj* pSchemaMap; // keep the retrieved schema info, to avoid the overhead by repeatly load schema + SDataFReader* pFileReader; // the file reader + SDelFReader* pDelFReader; // the del file reader + SArray* pDelIdx; // del file block index; + SBlockInfoBuf blockInfoBuf; + int32_t step; + STsdbReader* innerReader[2]; }; static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter); static int buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t endKey, int32_t capacity, STsdbReader* pReader); static TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader); -static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, STsdbReader* pReader, - SRowMerger* pMerger); +static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, STsdbReader* pReader); static int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts, SRowMerger* pMerger, SVersionRange* pVerRange, const char* id); -static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, SRowMerger* pMerger, - STsdbReader* pReader); +static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, STsdbReader* pReader); static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pTSRow, - STableBlockScanInfo* pInfo); + STableBlockScanInfo* pScanInfo); static int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, int32_t rowIndex); static void setComposedBlockFlag(STsdbReader* pReader, bool composed); @@ -214,7 +222,7 @@ static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* SVersionRange* pVerRange); static int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, - TSDBROW* pTSRow, STsdbReader* pReader, bool* freeTSRow); + TSDBROW* pResRow, STsdbReader* pReader, bool* freeTSRow); static int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, SRow** pTSRow); static int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key, @@ -235,7 +243,7 @@ static int32_t getInitialDelIndex(const SArray* pDelSkyline, int32_t order static STableBlockScanInfo* getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, const char* id); -static FORCE_INLINE STSchema* getLatestTableSchema(STsdbReader* pReader, uint64_t uid); +static STSchema* getLatestTableSchema(STsdbReader* pReader, uint64_t uid); static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } @@ -1889,7 +1897,7 @@ static bool tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SLastBlockReader* pLas return code; } -static FORCE_INLINE STSchema* getLatestTableSchema(STsdbReader* pReader, uint64_t uid) { +STSchema* getLatestTableSchema(STsdbReader* pReader, uint64_t uid) { if (pReader->pSchema != NULL) { return pReader->pSchema; } @@ -1912,6 +1920,12 @@ static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* terrno = code; return NULL; } + + code = tsdbRowMergerInit_rv(&pReader->status.merger, pReader->pSchema); + if (code != 0) { + terrno = code; + return NULL; + } } if (pReader->pSchema && sversion == pReader->pSchema->version) { @@ -1989,11 +2003,11 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (pReader->order == TSDB_ORDER_ASC) { if (minKey == key) { init = true; // todo check if pReader->pSchema is null or not - int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerAdd(&pReader->status.merger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); } if (minKey == tsLast) { @@ -2002,7 +2016,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, &fRow1, NULL); } else { init = true; - int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow1, pReader->pSchema); + int32_t code = tsdbRowMergerAdd(&pReader->status.merger, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2019,12 +2033,12 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, pRow, pSchema); } else { init = true; - int32_t code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema); + int32_t code = tsdbRowMergerAdd(&pReader->status.merger, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } } - int32_t code = doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + int32_t code = doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, pReader); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2037,12 +2051,12 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* return terrno; } - int32_t code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema); + int32_t code = tsdbRowMergerAdd(&pReader->status.merger, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } - code = doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + code = doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, pReader); if (code != TSDB_CODE_SUCCESS || merge.pTSchema == NULL) { return code; } @@ -2054,7 +2068,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, &fRow1, NULL); } else { init = true; - int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow1, pReader->pSchema); + int32_t code = tsdbRowMergerAdd(&pReader->status.merger, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2067,12 +2081,12 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, &fRow, NULL); } else { init = true; - int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerAdd(&pReader->status.merger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } } - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); } } @@ -2112,7 +2126,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, pBlockScanInfo->lastKey = tsLastBlock; return TSDB_CODE_SUCCESS; } else { - code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); + code = tsdbRowMergerAdd(&pReader->status.merger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2136,7 +2150,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, } } } else { // not merge block data - code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); + code = tsdbRowMergerAdd(&pReader->status.merger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2145,7 +2159,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, // merge with block data if ts == key if (tsLastBlock == pBlockData->aTSKEY[pDumpInfo->rowIndex]) { - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); } code = tsdbRowMergerGetRow(&merge, &pTSRow); @@ -2185,22 +2199,22 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader if (key < ts) { // imem, mem are all empty, file blocks (data blocks and last block) exist return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader); } else if (key == ts) { - SRow* pTSRow = NULL; - SRowMerger merge = {0}; + SRow* pTSRow = NULL; + SRowMerger* pMerger = &pReader->status.merger; - int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerAdd(pMerger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tsdbRowMergerAdd(&merge, &fRow1, NULL); + tsdbRowMergerAdd(pMerger, &fRow1, NULL); - doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, ts, &merge, &pReader->verRange, pReader->idStr); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, ts, pMerger, &pReader->verRange, pReader->idStr); - code = tsdbRowMergerGetRow(&merge, &pTSRow); + code = tsdbRowMergerGetRow(pMerger, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2208,7 +2222,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tsdbRowMergerClear(&merge); + tsdbRowMergerClear_rv(pMerger); return code; } else { return TSDB_CODE_SUCCESS; @@ -2296,12 +2310,12 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { init = true; TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); + code = tsdbRowMergerAdd(&pReader->status.merger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); } if (minKey == tsLast) { @@ -2310,7 +2324,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, &fRow1, NULL); } else { init = true; - code = tsdbRowMergerInit(&merge, NULL, &fRow1, pReader->pSchema); + code = tsdbRowMergerAdd(&pReader->status.merger, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2324,14 +2338,13 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, piRow, piSchema); } else { init = true; - code = tsdbRowMergerInit(&merge, pSchema, piRow, piSchema); + code = tsdbRowMergerAdd(&pReader->status.merger, piRow, piSchema); if (code != TSDB_CODE_SUCCESS) { return code; } } - code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, - pReader); + code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, pReader); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2346,13 +2359,12 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, pRow, pSchema); } else { // STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema); + code = tsdbRowMergerAdd(&pReader->status.merger, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } } - code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, - pReader); + code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, pReader); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2360,13 +2372,12 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } else { if (minKey == k.ts) { init = true; - code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema); + code = tsdbRowMergerAdd(&pReader->status.merger, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } - code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, - pReader); + code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, pReader); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2377,14 +2388,12 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, piRow, piSchema); } else { init = true; - // STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); - code = tsdbRowMergerInit(&merge, pSchema, piRow, piSchema); + code = tsdbRowMergerAdd(&pReader->status.merger, piRow, piSchema); if (code != TSDB_CODE_SUCCESS) { return code; } } - code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, - pReader); + code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, pReader); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2396,7 +2405,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, &fRow1, NULL); } else { init = true; - code = tsdbRowMergerInit(&merge, NULL, &fRow1, pReader->pSchema); + code = tsdbRowMergerAdd(&pReader->status.merger, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2407,7 +2416,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); if (!init) { - code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); + code = tsdbRowMergerAdd(&pReader->status.merger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2417,7 +2426,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } tsdbRowMergerAdd(&merge, &fRow, NULL); } - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); } } @@ -2598,15 +2607,13 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); SRow* pTSRow = NULL; - SRowMerger merge = {0}; - - code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); + code = tsdbRowMergerAdd(&pReader->status.merger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); - code = tsdbRowMergerGetRow(&merge, &pTSRow); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); + code = tsdbRowMergerGetRow(&pReader->status.merger, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2614,7 +2621,7 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tsdbRowMergerClear(&merge); + tsdbRowMergerClear_rv(&pReader->status.merger); return code; } } @@ -3685,8 +3692,9 @@ TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* p } } -int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, SRowMerger* pMerger, - STsdbReader* pReader) { +int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, STsdbReader* pReader) { + SRowMerger* pMerger = &pReader->status.merger; + while (1) { pIter->hasVal = tsdbTbDataIterNext(pIter->iter); if (!pIter->hasVal) { @@ -3765,10 +3773,10 @@ static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanIn return code; } -int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, STsdbReader* pReader, - SRowMerger* pMerger) { +int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, STsdbReader* pReader) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + SRowMerger* pMerger = &pReader->status.merger; bool asc = ASCENDING_TRAVERSE(pReader->order); int64_t key = pBlockData->aTSKEY[pDumpInfo->rowIndex]; int32_t step = asc ? 1 : -1; @@ -3847,7 +3855,6 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, } } - SRowMerger merge = {0}; terrno = 0; int32_t code = 0; @@ -3859,8 +3866,7 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, return terrno; } - STSchema* ps = (pReader->pSchema != NULL)? pReader->pSchema:pTSchema; - code = tsdbRowMergerInit(&merge, ps, ¤t, pTSchema); + code = tsdbRowMergerAdd(&pReader->status.merger, ¤t, pTSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3870,28 +3876,28 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, return terrno; } - tsdbRowMergerAdd(&merge, pNextRow, pTSchema1); + tsdbRowMergerAdd(&pReader->status.merger,pNextRow, pTSchema1); } else { // let's merge rows in file block - code = tsdbRowMergerInit(&merge, NULL, ¤t, pReader->pSchema); + code = tsdbRowMergerAdd(&pReader->status.merger, ¤t, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } - tsdbRowMergerAdd(&merge, pNextRow, NULL); + tsdbRowMergerAdd(&pReader->status.merger,pNextRow, NULL); } - code = doMergeRowsInBuf(pIter, uid, TSDBROW_TS(¤t), pDelList, &merge, pReader); + code = doMergeRowsInBuf(pIter, uid, TSDBROW_TS(¤t), pDelList, pReader); if (code != TSDB_CODE_SUCCESS) { return code; } - code = tsdbRowMergerGetRow(&merge, &pResRow->pTSRow); + code = tsdbRowMergerGetRow(&pReader->status.merger, &pResRow->pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } pResRow->type = TSDBROW_ROW_FMT; - tsdbRowMergerClear(&merge); + tsdbRowMergerClear_rv(&pReader->status.merger); *freeTSRow = true; return TSDB_CODE_SUCCESS; @@ -3899,7 +3905,7 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, SRow** pTSRow) { - SRowMerger merge = {0}; + SRowMerger* pMerger = &pReader->status.merger; TSDBKEY k = TSDBROW_KEY(pRow); TSDBKEY ik = TSDBROW_KEY(piRow); @@ -3914,46 +3920,43 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p } if (ASCENDING_TRAVERSE(pReader->order)) { // ascending order imem --> mem - int32_t code = tsdbRowMergerInit(&merge, pSchema, piRow, piSchema); + int32_t code = tsdbRowMergerAdd(&pReader->status.merger, piRow, piSchema); if (code != TSDB_CODE_SUCCESS) { return code; } - code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, - pReader); + code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, pReader); if (code != TSDB_CODE_SUCCESS) { return code; } - tsdbRowMergerAdd(&merge, pRow, pSchema); + tsdbRowMergerAdd(&pReader->status.merger,pRow, pSchema); code = - doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, pReader); if (code != TSDB_CODE_SUCCESS) { return code; } } else { - int32_t code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema); - if (code != TSDB_CODE_SUCCESS || merge.pTSchema == NULL) { + int32_t code = tsdbRowMergerAdd(&pReader->status.merger, pRow, pSchema); + if (code != TSDB_CODE_SUCCESS || pMerger->pTSchema == NULL) { return code; } - code = - doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, pReader); if (code != TSDB_CODE_SUCCESS) { return code; } - tsdbRowMergerAdd(&merge, piRow, piSchema); - code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, - pReader); + tsdbRowMergerAdd(&pReader->status.merger, piRow, piSchema); + code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, pReader); if (code != TSDB_CODE_SUCCESS) { return code; } } - int32_t code = tsdbRowMergerGetRow(&merge, pTSRow); - tsdbRowMergerClear(&merge); + int32_t code = tsdbRowMergerGetRow(pMerger, pTSRow); + tsdbRowMergerClear_rv(pMerger); return code; } @@ -4334,6 +4337,10 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL } } + if (pReader->pSchema != NULL) { + tsdbRowMergerInit_rv(&pReader->status.merger, pReader->pSchema); + } + pReader->pSchemaMap = tSimpleHashInit(8, taosFastHash); if (pReader->pSchemaMap == NULL) { tsdbError("failed init schema hash for reader %s", pReader->idStr); @@ -4483,6 +4490,8 @@ void tsdbReaderClose(STsdbReader* pReader) { pCost->initDelSkylineIterTime, pReader->idStr); taosMemoryFree(pReader->idStr); + + tsdbRowMergerCleanup_rv(&pReader->status.merger); taosMemoryFree(pReader->pSchema); tSimpleHashCleanup(pReader->pSchemaMap); diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 8e778da877..909d354c38 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -778,58 +778,40 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) pMerger->version = key.version; return code; } -/* -int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { - int32_t code = 0; - TSDBKEY key = TSDBROW_KEY(pRow); - SColVal *pColVal = &(SColVal){0}; - STColumn *pTColumn; - pMerger->pTSchema = pTSchema; - pMerger->version = key.version; - - pMerger->pArray = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)); +int32_t tsdbRowMergerInit_rv(SRowMerger* pMerger, STSchema *pSchema) { + pMerger->pTSchema = pSchema; + pMerger->pArray = taosArrayInit(pSchema->numOfCols, sizeof(SColVal)); if (pMerger->pArray == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; + return TSDB_CODE_OUT_OF_MEMORY; + } else { + return TSDB_CODE_SUCCESS; } - - // ts - pTColumn = &pTSchema->columns[0]; - - ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP); - - *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = key.ts}); - if (taosArrayPush(pMerger->pArray, pColVal) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - - // other - for (int16_t iCol = 1; iCol < pTSchema->numOfCols; iCol++) { - tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal); - if ((!COL_VAL_IS_NONE(pColVal)) && (!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) { - uint8_t *pVal = pColVal->value.pData; - - pColVal->value.pData = NULL; - code = tRealloc(&pColVal->value.pData, pColVal->value.nData); - if (code) goto _exit; - - if (pColVal->value.nData) { - memcpy(pColVal->value.pData, pVal, pColVal->value.nData); - } - } - - if (taosArrayPush(pMerger->pArray, pColVal) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - } - -_exit: - return code; } -*/ + +void tsdbRowMergerClear_rv(SRowMerger* pMerger) { + for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) { + SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol); + if (IS_VAR_DATA_TYPE(pTColVal->type)) { + tFree(pTColVal->value.pData); + } + } + + taosArrayClear(pMerger->pArray); +} + +void tsdbRowMergerCleanup_rv(SRowMerger* pMerger) { + int32_t numOfCols = taosArrayGetSize(pMerger->pArray); + for (int32_t iCol = 1; iCol < numOfCols; iCol++) { + SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol); + if (IS_VAR_DATA_TYPE(pTColVal->type)) { + tFree(pTColVal->value.pData); + } + } + + taosArrayDestroy(pMerger->pArray); +} + void tsdbRowMergerClear(SRowMerger *pMerger) { for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) { SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol); From 9a14e0db47909ce9d141f3a4f5d6cf08c3efb832 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Apr 2023 00:30:43 +0800 Subject: [PATCH 084/109] fix(query): fix error in add row into merger. --- source/dnode/vnode/src/tsdb/tsdbUtil.c | 142 +++++++++++++++++-------- 1 file changed, 99 insertions(+), 43 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 909d354c38..7d1cca37b5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -712,71 +712,127 @@ _exit: int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); - SColVal *pColVal = &(SColVal){0}; + SColVal * pColVal = &(SColVal){0}; STColumn *pTColumn; int32_t iCol, jCol = 1; if (NULL == pTSchema) { pTSchema = pMerger->pTSchema; } - ASSERT(((SColVal *)pMerger->pArray->pData)->value.val == key.ts); - for (iCol = 1; iCol < pMerger->pTSchema->numOfCols && jCol < pTSchema->numOfCols; ++iCol) { - pTColumn = &pMerger->pTSchema->columns[iCol]; - if (pTSchema->columns[jCol].colId < pTColumn->colId) { - ++jCol; - --iCol; - continue; - } else if (pTSchema->columns[jCol].colId > pTColumn->colId) { - continue; + if (taosArrayGetSize(pMerger->pArray) == 0) { + // ts + pTColumn = &pTSchema->columns[jCol++]; + + ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP); + + *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = key.ts}); + if (taosArrayPush(pMerger->pArray, pColVal) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + return code; + // goto _exit; } - tsdbRowGetColVal(pRow, pTSchema, jCol++, pColVal); + // other + for (iCol = 1; jCol < pTSchema->numOfCols && iCol < pMerger->pTSchema->numOfCols; ++iCol) { + pTColumn = &pMerger->pTSchema->columns[iCol]; + if (pTSchema->columns[jCol].colId < pTColumn->colId) { + ++jCol; + --iCol; + continue; + } else if (pTSchema->columns[jCol].colId > pTColumn->colId) { + taosArrayPush(pMerger->pArray, &COL_VAL_NONE(pTColumn->colId, pTColumn->type)); + continue; + } - if (key.version > pMerger->version) { - if (!COL_VAL_IS_NONE(pColVal)) { - if (IS_VAR_DATA_TYPE(pColVal->type)) { - SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol); - if (!COL_VAL_IS_NULL(pColVal)) { - code = tRealloc(&pTColVal->value.pData, pColVal->value.nData); - if (code) return code; + tsdbRowGetColVal(pRow, pTSchema, jCol++, pColVal); + if ((!COL_VAL_IS_NONE(pColVal)) && (!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) { + uint8_t *pVal = pColVal->value.pData; - pTColVal->value.nData = pColVal->value.nData; - if (pTColVal->value.nData) { - memcpy(pTColVal->value.pData, pColVal->value.pData, pTColVal->value.nData); + pColVal->value.pData = NULL; + code = tRealloc(&pColVal->value.pData, pColVal->value.nData); + if (code) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + if (pColVal->value.nData) { + memcpy(pColVal->value.pData, pVal, pColVal->value.nData); + } + } + + if (taosArrayPush(pMerger->pArray, pColVal) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + return code; + } + } + + for (; iCol < pMerger->pTSchema->numOfCols; ++iCol) { + pTColumn = &pMerger->pTSchema->columns[iCol]; + taosArrayPush(pMerger->pArray, &COL_VAL_NONE(pTColumn->colId, pTColumn->type)); + } + + pMerger->version = key.version; + return 0; + } else { + ASSERT(((SColVal *)pMerger->pArray->pData)->value.val == key.ts); + + for (iCol = 1; iCol < pMerger->pTSchema->numOfCols && jCol < pTSchema->numOfCols; ++iCol) { + pTColumn = &pMerger->pTSchema->columns[iCol]; + if (pTSchema->columns[jCol].colId < pTColumn->colId) { + ++jCol; + --iCol; + continue; + } else if (pTSchema->columns[jCol].colId > pTColumn->colId) { + continue; + } + + tsdbRowGetColVal(pRow, pTSchema, jCol++, pColVal); + + if (key.version > pMerger->version) { + if (!COL_VAL_IS_NONE(pColVal)) { + if (IS_VAR_DATA_TYPE(pColVal->type)) { + SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol); + if (!COL_VAL_IS_NULL(pColVal)) { + code = tRealloc(&pTColVal->value.pData, pColVal->value.nData); + if (code) return code; + + pTColVal->value.nData = pColVal->value.nData; + if (pTColVal->value.nData) { + memcpy(pTColVal->value.pData, pColVal->value.pData, pTColVal->value.nData); + } + pTColVal->flag = 0; + } else { + tFree(pTColVal->value.pData); + taosArraySet(pMerger->pArray, iCol, pColVal); } - pTColVal->flag = 0; } else { - tFree(pTColVal->value.pData); taosArraySet(pMerger->pArray, iCol, pColVal); } - } else { - taosArraySet(pMerger->pArray, iCol, pColVal); } - } - } else if (key.version < pMerger->version) { - SColVal *tColVal = (SColVal *)taosArrayGet(pMerger->pArray, iCol); - if (COL_VAL_IS_NONE(tColVal) && !COL_VAL_IS_NONE(pColVal)) { - if ((!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) { - code = tRealloc(&tColVal->value.pData, pColVal->value.nData); - if (code) return code; + } else if (key.version < pMerger->version) { + SColVal *tColVal = (SColVal *)taosArrayGet(pMerger->pArray, iCol); + if (COL_VAL_IS_NONE(tColVal) && !COL_VAL_IS_NONE(pColVal)) { + if ((!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) { + code = tRealloc(&tColVal->value.pData, pColVal->value.nData); + if (code) return code; - tColVal->value.nData = pColVal->value.nData; - if (pColVal->value.nData) { - memcpy(tColVal->value.pData, pColVal->value.pData, pColVal->value.nData); + tColVal->value.nData = pColVal->value.nData; + if (pColVal->value.nData) { + memcpy(tColVal->value.pData, pColVal->value.pData, pColVal->value.nData); + } + tColVal->flag = 0; + } else { + taosArraySet(pMerger->pArray, iCol, pColVal); } - tColVal->flag = 0; - } else { - taosArraySet(pMerger->pArray, iCol, pColVal); } + } else { + ASSERT(0 && "dup versions not allowed"); } - } else { - ASSERT(0 && "dup versions not allowed"); } - } - pMerger->version = key.version; - return code; + pMerger->version = key.version; + return code; + } } int32_t tsdbRowMergerInit_rv(SRowMerger* pMerger, STSchema *pSchema) { From 37d042a338264df27435c1b6dcc8c15287413e6a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Apr 2023 00:33:24 +0800 Subject: [PATCH 085/109] fix(query): fix error in add row into merger. --- source/dnode/vnode/src/tsdb/tsdbUtil.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 7d1cca37b5..0e2fb4b6aa 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -722,6 +722,7 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) if (taosArrayGetSize(pMerger->pArray) == 0) { // ts + jCol = 0; pTColumn = &pTSchema->columns[jCol++]; ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP); From a73957aabadb33762ca72c9917336ffe2fb494f9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Apr 2023 09:24:15 +0800 Subject: [PATCH 086/109] fix(query): fix error in add row into merger. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 85 +++++++++++--------------- 1 file changed, 37 insertions(+), 48 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index dd6913039c..0bba83622c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1954,7 +1954,7 @@ static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, TSDBROW* pRow, SIterInfo* pIter, int64_t key, SLastBlockReader* pLastBlockReader) { - SRowMerger merge = {0}; + SRowMerger* pMerger = &pReader->status.merger; SRow* pTSRow = NULL; SBlockData* pBlockData = &pReader->status.fileBlockData; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; @@ -2013,7 +2013,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tsdbRowMergerAdd(&merge, &fRow1, NULL); + tsdbRowMergerAdd(&pReader->status.merger, &fRow1, NULL); } else { init = true; int32_t code = tsdbRowMergerAdd(&pReader->status.merger, &fRow1, pReader->pSchema); @@ -2021,7 +2021,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* return code; } } - doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge, &pReader->verRange, pReader->idStr); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &pReader->status.merger, &pReader->verRange, pReader->idStr); } if (minKey == k.ts) { @@ -2030,7 +2030,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* return terrno; } if (init) { - tsdbRowMergerAdd(&merge, pRow, pSchema); + tsdbRowMergerAdd(pMerger, pRow, pSchema); } else { init = true; int32_t code = tsdbRowMergerAdd(&pReader->status.merger, pRow, pSchema); @@ -2057,7 +2057,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* } code = doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, pReader); - if (code != TSDB_CODE_SUCCESS || merge.pTSchema == NULL) { + if (code != TSDB_CODE_SUCCESS || pMerger->pTSchema == NULL) { return code; } } @@ -2065,7 +2065,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tsdbRowMergerAdd(&merge, &fRow1, NULL); + tsdbRowMergerAdd(pMerger, &fRow1, NULL); } else { init = true; int32_t code = tsdbRowMergerAdd(&pReader->status.merger, &fRow1, pReader->pSchema); @@ -2073,12 +2073,12 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* return code; } } - doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge, &pReader->verRange, pReader->idStr); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->verRange, pReader->idStr); } if (minKey == key) { if (init) { - tsdbRowMergerAdd(&merge, &fRow, NULL); + tsdbRowMergerAdd(pMerger, &fRow, NULL); } else { init = true; int32_t code = tsdbRowMergerAdd(&pReader->status.merger, &fRow, pReader->pSchema); @@ -2090,7 +2090,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* } } - int32_t code = tsdbRowMergerGetRow(&merge, &pTSRow); + int32_t code = tsdbRowMergerGetRow(pMerger, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2098,7 +2098,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tsdbRowMergerClear(&merge); + tsdbRowMergerClear_rv(pMerger); return code; } @@ -2106,12 +2106,12 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, bool mergeBlockData) { + SRowMerger* pMerger = &pReader->status.merger; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); bool copied = false; int32_t code = TSDB_CODE_SUCCESS; SRow* pTSRow = NULL; - SRowMerger merge = {0}; TSDBROW fRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", %s", fRow.pBlockData, fRow.iRow, pLastBlockReader->uid, pReader->idStr); @@ -2132,10 +2132,10 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, } TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tsdbRowMergerAdd(&merge, &fRow1, NULL); - doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge, &pReader->verRange, pReader->idStr); + tsdbRowMergerAdd(pMerger, &fRow1, NULL); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, pMerger, &pReader->verRange, pReader->idStr); - code = tsdbRowMergerGetRow(&merge, &pTSRow); + code = tsdbRowMergerGetRow(pMerger, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2143,7 +2143,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tsdbRowMergerClear(&merge); + tsdbRowMergerClear_rv(pMerger); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2155,14 +2155,14 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, return code; } - doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge, &pReader->verRange, pReader->idStr); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, pMerger, &pReader->verRange, pReader->idStr); // merge with block data if ts == key if (tsLastBlock == pBlockData->aTSKEY[pDumpInfo->rowIndex]) { doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); } - code = tsdbRowMergerGetRow(&merge, &pTSRow); + code = tsdbRowMergerGetRow(pMerger, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2170,7 +2170,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tsdbRowMergerClear(&merge); + tsdbRowMergerClear_rv(pMerger); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2237,7 +2237,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, SLastBlockReader* pLastBlockReader) { - SRowMerger merge = {0}; + SRowMerger* pMerger = &pReader->status.merger; SRow* pTSRow = NULL; int32_t code = TSDB_CODE_SUCCESS; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; @@ -2310,7 +2310,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { init = true; TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - code = tsdbRowMergerAdd(&pReader->status.merger, &fRow, pReader->pSchema); + code = tsdbRowMergerAdd(pMerger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2321,24 +2321,24 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tsdbRowMergerAdd(&merge, &fRow1, NULL); + tsdbRowMergerAdd(pMerger, &fRow1, NULL); } else { init = true; - code = tsdbRowMergerAdd(&pReader->status.merger, &fRow1, pReader->pSchema); + code = tsdbRowMergerAdd(pMerger, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } } - doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge, &pReader->verRange, pReader->idStr); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->verRange, pReader->idStr); } if (minKey == ik.ts) { if (init) { - tsdbRowMergerAdd(&merge, piRow, piSchema); + tsdbRowMergerAdd(pMerger, piRow, piSchema); } else { init = true; - code = tsdbRowMergerAdd(&pReader->status.merger, piRow, piSchema); + code = tsdbRowMergerAdd(pMerger, piRow, piSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2352,14 +2352,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == k.ts) { if (init) { - if (merge.pTSchema == NULL) { - return code; - } - - tsdbRowMergerAdd(&merge, pRow, pSchema); + tsdbRowMergerAdd(pMerger, pRow, pSchema); } else { // STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - code = tsdbRowMergerAdd(&pReader->status.merger, pRow, pSchema); + code = tsdbRowMergerAdd(pMerger, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2372,7 +2368,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } else { if (minKey == k.ts) { init = true; - code = tsdbRowMergerAdd(&pReader->status.merger, pRow, pSchema); + code = tsdbRowMergerAdd(pMerger, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2385,10 +2381,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == ik.ts) { if (init) { - tsdbRowMergerAdd(&merge, piRow, piSchema); + tsdbRowMergerAdd(pMerger, piRow, piSchema); } else { init = true; - code = tsdbRowMergerAdd(&pReader->status.merger, piRow, piSchema); + code = tsdbRowMergerAdd(pMerger, piRow, piSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2402,39 +2398,32 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tsdbRowMergerAdd(&merge, &fRow1, NULL); + tsdbRowMergerAdd(pMerger, &fRow1, NULL); } else { init = true; - code = tsdbRowMergerAdd(&pReader->status.merger, &fRow1, pReader->pSchema); + code = tsdbRowMergerAdd(pMerger, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } } - doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge, &pReader->verRange, pReader->idStr); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->verRange, pReader->idStr); } if (minKey == key) { TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); if (!init) { - code = tsdbRowMergerAdd(&pReader->status.merger, &fRow, pReader->pSchema); + code = tsdbRowMergerAdd(pMerger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } } else { - if (merge.pTSchema == NULL) { - return code; - } - tsdbRowMergerAdd(&merge, &fRow, NULL); + tsdbRowMergerAdd(pMerger, &fRow, NULL); } doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); } } - if (merge.pTSchema == NULL) { - return code; - } - - code = tsdbRowMergerGetRow(&merge, &pTSRow); + code = tsdbRowMergerGetRow(pMerger, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2442,7 +2431,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tsdbRowMergerClear(&merge); + tsdbRowMergerClear_rv(pMerger); return code; } From ae8a9d25c019b9b5711f66b05b2e4cf2d57be980 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Apr 2023 09:41:49 +0800 Subject: [PATCH 087/109] refactor: do some internal refactor. --- source/dnode/vnode/src/inc/tsdb.h | 2 +- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index b2b04abff1..0de1d7de61 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -769,13 +769,13 @@ typedef struct SLDataIter { bool ignoreEarlierTs; } SLDataIter; +#define tMergeTreeGetRow(_t) ((_t)->pIter->rInfo.row) int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo, bool destroyLoadInfo, const char *idStr, bool strictTimeRange, SLDataIter* pLDataIter); void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter); bool tMergeTreeNext(SMergeTree *pMTree); bool tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree); -TSDBROW tMergeTreeGetRow(SMergeTree *pMTree); void tMergeTreeClose(SMergeTree *pMTree); SSttBlockLoadInfo *tCreateLastBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols, int32_t numOfStt); diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index e354a50253..5a1b9d0ce4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -647,8 +647,6 @@ bool tMergeTreeNext(SMergeTree *pMTree) { return pMTree->pIter != NULL; } -TSDBROW tMergeTreeGetRow(SMergeTree *pMTree) { return pMTree->pIter->rInfo.row; } - void tMergeTreeClose(SMergeTree *pMTree) { pMTree->pIter = NULL; if (pMTree->destroyLoadInfo) { From d57676c924870faf5dcc5be9a42d699b2d0cbf46 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 25 Apr 2023 09:52:16 +0800 Subject: [PATCH 088/109] fix: delete table error --- source/dnode/vnode/src/meta/metaTable.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 2b5bc36acb..96eec89127 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -936,8 +936,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { int tLen = 0; if (tdbTbGet(pMeta->pUidIdx, &e.ctbEntry.suid, sizeof(tb_uid_t), &tData, &tLen) == 0) { - version = ((SUidIdxVal *)tData)[0].version; - STbDbKey tbDbKey = {.uid = e.ctbEntry.suid, .version = version}; + STbDbKey tbDbKey = {.uid = e.ctbEntry.suid, .version = ((SUidIdxVal *)tData)[0].version}; if (tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &tData, &tLen) == 0) { SDecoder tdc = {0}; SMetaEntry stbEntry = {0}; From e724bc15e03f9b5bc9cb06f7b3a35bdc39a6e6e9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Apr 2023 09:54:00 +0800 Subject: [PATCH 089/109] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 0bba83622c..a04100343a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2567,8 +2567,8 @@ static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScan } static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader) { - TSDBROW row = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - return TSDBROW_TS(&row); + TSDBROW* pRow = &tMergeTreeGetRow((&(pLastBlockReader)->mergeTree)); + return pRow->pBlockData->aTSKEY[pRow->iRow]; } static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { return pLastBlockReader->mergeTree.pIter != NULL; } From 4c4e1486352638d0fe2df5763f4617aa1c1f00a5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Apr 2023 10:19:37 +0800 Subject: [PATCH 090/109] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 45 ++++++++++++++------------ 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index a04100343a..90bb391120 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -891,8 +891,7 @@ static void cleanupTableScanInfo(SSHashObj* pTableMap) { } } -static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockNumber* pBlockNum) { - int32_t numOfQTable = 0; +static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockNumber* pBlockNum, SArray* pTableScanInfoList) { size_t sizeInDisk = 0; size_t numOfTables = taosArrayGetSize(pIndexList); @@ -952,7 +951,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN } if (taosArrayGetSize(pScanInfo->pBlockList) > 0) { - numOfQTable += 1; + taosArrayPush(pTableScanInfoList, &pScanInfo); } } @@ -963,8 +962,8 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN tsdbDebug( "load block of %ld tables completed, blocks:%d in %d tables, last-files:%d, block-info-size:%.2f Kb, elapsed " "time:%.2f ms %s", - numOfTables, pBlockNum->numOfBlocks, numOfQTable, pBlockNum->numOfLastFiles, sizeInDisk / 1000.0, el, - pReader->idStr); + numOfTables, pBlockNum->numOfBlocks, (int32_t)taosArrayGetSize(pTableScanInfoList), pBlockNum->numOfLastFiles, + sizeInDisk / 1000.0, el, pReader->idStr); pReader->cost.numOfBlocks += total; pReader->cost.headFileLoadTime += el; @@ -1443,7 +1442,7 @@ static int32_t doSetCurrentBlock(SDataBlockIter* pBlockIter, const char* idStr) return TSDB_CODE_SUCCESS; } -static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks) { +static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks, SArray* pTableList) { bool asc = ASCENDING_TRAVERSE(pReader->order); SBlockOrderSupporter sup = {0}; @@ -1452,7 +1451,8 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte pBlockIter->pTableMap = pReader->status.pTableMap; // access data blocks according to the offset of each block in asc/desc order. - int32_t numOfTables = (int32_t)tSimpleHashGetSize(pReader->status.pTableMap); +// int32_t numOfTables = (int32_t)tSimpleHashGetSize(pReader->status.pTableMap); + int32_t numOfTables = taosArrayGetSize(pTableList); int64_t st = taosGetTimestampUs(); int32_t code = initBlockOrderSupporter(&sup, numOfTables); @@ -1464,16 +1464,18 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte void* ptr = NULL; int32_t iter = 0; - while (1) { - ptr = tSimpleHashIterate(pReader->status.pTableMap, ptr, &iter); - if (ptr == NULL) { - break; - } +// while (1) { +// ptr = tSimpleHashIterate(pReader->status.pTableMap, ptr, &iter); +// if (ptr == NULL) { +// break; +// } - STableBlockScanInfo* pTableScanInfo = *(STableBlockScanInfo**)ptr; - if (pTableScanInfo->pBlockList == NULL || taosArrayGetSize(pTableScanInfo->pBlockList) == 0) { - continue; - } + for(int32_t i = 0; i < numOfTables; ++i) { + STableBlockScanInfo* pTableScanInfo = taosArrayGetP(pTableList, i); + ASSERT(pTableScanInfo->pBlockList != NULL && taosArrayGetSize(pTableScanInfo->pBlockList) > 0); +// if (pTableScanInfo->pBlockList == NULL || taosArrayGetSize(pTableScanInfo->pBlockList) == 0) { +// continue; +// } size_t num = taosArrayGetSize(pTableScanInfo->pBlockList); sup.numOfBlocksPerTable[sup.numOfTables] = num; @@ -2914,7 +2916,7 @@ TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader) } } -static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { +static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SArray* pTableList) { SReaderStatus* pStatus = &pReader->status; pBlockNum->numOfBlocks = 0; pBlockNum->numOfLastFiles = 0; @@ -2942,13 +2944,14 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { } if (taosArrayGetSize(pIndexList) > 0 || pReader->pFileReader->pSet->nSttF > 0) { - code = doLoadFileBlock(pReader, pIndexList, pBlockNum); + code = doLoadFileBlock(pReader, pIndexList, pBlockNum, pTableList); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(pIndexList); return code; } if (pBlockNum->numOfBlocks + pBlockNum->numOfLastFiles > 0) { + ASSERT(taosArrayGetSize(pTableList) > 0); break; } } @@ -3353,7 +3356,9 @@ static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBlockIter) { SBlockNumber num = {0}; - int32_t code = moveToNextFile(pReader, &num); + SArray* pTableList = taosArrayInit(40, POINTER_BYTES); + + int32_t code = moveToNextFile(pReader, &num, pTableList); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3366,7 +3371,7 @@ static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBl // initialize the block iterator for a new fileset if (num.numOfBlocks > 0) { - code = initBlockIterator(pReader, pBlockIter, num.numOfBlocks); + code = initBlockIterator(pReader, pBlockIter, num.numOfBlocks, pTableList); } else { // no block data, only last block exists tBlockDataReset(&pReader->status.fileBlockData); resetDataBlockIterator(pBlockIter, pReader->order); From 6467b99a52299ac076495f553eccd792c7c5cc07 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Apr 2023 10:23:21 +0800 Subject: [PATCH 091/109] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 90bb391120..8d4708925b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1461,8 +1461,8 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte } int32_t cnt = 0; - void* ptr = NULL; - int32_t iter = 0; +// void* ptr = NULL; +// int32_t iter = 0; // while (1) { // ptr = tSimpleHashIterate(pReader->status.pTableMap, ptr, &iter); @@ -2951,7 +2951,7 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SAr } if (pBlockNum->numOfBlocks + pBlockNum->numOfLastFiles > 0) { - ASSERT(taosArrayGetSize(pTableList) > 0); +// ASSERT(taosArrayGetSize(pTableList) > 0); break; } } From d8762b9bc956782904505638ce5af23549d16999 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Apr 2023 10:48:21 +0800 Subject: [PATCH 092/109] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 6 ++-- source/dnode/vnode/src/tsdb/tsdbRead.c | 32 ++++++++++++++++----- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 5a1b9d0ce4..79f4a17f65 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -261,8 +261,10 @@ int32_t tLDataIterOpen(struct SLDataIter *pIter, SDataFReader *pReader, int32_t pIter->pReader = pReader; pIter->iStt = iStt; pIter->backward = backward; - pIter->verRange = *pRange; - pIter->timeWindow = *pTimeWindow; + pIter->verRange.minVer = pRange->minVer; + pIter->verRange.maxVer = pRange->maxVer; + pIter->timeWindow.skey = pTimeWindow->skey; + pIter->timeWindow.ekey = pTimeWindow->ekey; pIter->pBlockLoadInfo = pBlockLoadInfo; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 8d4708925b..efd8ab1288 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -148,6 +148,7 @@ typedef struct STableUidList { typedef struct SReaderStatus { bool loadFromFile; // check file stage bool composedDataBlock; // the returned data block is a composed block or not + bool mapDataCleaned; // mapData has been cleaned up alreay or not SSHashObj* pTableMap; // SHash STableBlockScanInfo** pTableIter; // table iterator used in building in-memory buffer data blocks. STableUidList uidList; // check tables in uid order, to avoid the repeatly load of blocks in STT. @@ -875,7 +876,18 @@ _end: return code; } -static void cleanupTableScanInfo(SSHashObj* pTableMap) { +static void doCleanupTableScanInfo(STableBlockScanInfo* pScanInfo) { + // reset the index in last block when handing a new file + tMapDataClear(&pScanInfo->mapData); + taosArrayClear(pScanInfo->pBlockList); +} + +static void cleanupTableScanInfo(SReaderStatus* pStatus) { + if (pStatus->mapDataCleaned) { + return; + } + + SSHashObj* pTableMap = pStatus->pTableMap; STableBlockScanInfo** px = NULL; int32_t iter = 0; @@ -885,10 +897,10 @@ static void cleanupTableScanInfo(SSHashObj* pTableMap) { break; } - // reset the index in last block when handing a new file - tMapDataClear(&(*px)->mapData); - taosArrayClear((*px)->pBlockList); + doCleanupTableScanInfo(*px); } + + pStatus->mapDataCleaned = true; } static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockNumber* pBlockNum, SArray* pTableScanInfoList) { @@ -896,8 +908,10 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN size_t numOfTables = taosArrayGetSize(pIndexList); int64_t st = taosGetTimestampUs(); - cleanupTableScanInfo(pReader->status.pTableMap); + cleanupTableScanInfo(&pReader->status); + // set the flag for the new file + pReader->status.mapDataCleaned = false; for (int32_t i = 0; i < numOfTables; ++i) { SBlockIdx* pBlockIdx = taosArrayGet(pIndexList, i); STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, pBlockIdx->uid, pReader->idStr); @@ -3022,8 +3036,12 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { // load the last data block of current table STableBlockScanInfo* pScanInfo = *(STableBlockScanInfo**)pStatus->pTableIter; - bool hasVal = initLastBlockReader(pLastBlockReader, pScanInfo, pReader); - if (!hasVal) { + // reset the index in last block when handing a new file + doCleanupTableScanInfo(pScanInfo); + pStatus->mapDataCleaned = true; + + bool hasDataInLastFile = initLastBlockReader(pLastBlockReader, pScanInfo, pReader); + if (!hasDataInLastFile) { bool hasNexTable = moveToNextTable(pUidList, pStatus); if (!hasNexTable) { return TSDB_CODE_SUCCESS; From f264f29bbda2f238d782259178f5f611794942f1 Mon Sep 17 00:00:00 2001 From: haoranchen Date: Tue, 25 Apr 2023 11:17:49 +0800 Subject: [PATCH 093/109] fix: restore ci buildPath --- tests/parallel_test/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/run.sh b/tests/parallel_test/run.sh index de343752c6..f05e0dfc83 100755 --- a/tests/parallel_test/run.sh +++ b/tests/parallel_test/run.sh @@ -303,7 +303,7 @@ function run_thread() { if [ ! -z "$corefile" ]; then echo -e "\e[34m corefiles: $corefile \e[0m" local build_dir=$log_dir/build_${hosts[index]} - local remote_build_dir="${workdirs[index]}/{DEBUGPATH}/build" + local remote_build_dir="${workdirs[index]}/${DEBUGPATH}/build" # if [ $ent -ne 0 ]; then # remote_build_dir="${workdirs[index]}/{DEBUGPATH}/build" # fi From 05d16920734367eeb29173febd4716930e9477ad Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Apr 2023 13:50:05 +0800 Subject: [PATCH 094/109] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index efd8ab1288..8e70badbc1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -18,6 +18,7 @@ #include "tsimplehash.h" #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) +#define getCurrentKeyInLastBlock(_r) ((_r)->currentKey) typedef enum { EXTERNAL_ROWS_PREV = 0x1, @@ -108,6 +109,7 @@ typedef struct SLastBlockReader { uint64_t uid; SMergeTree mergeTree; SSttBlockLoadInfo* pInfo; + int64_t currentKey; } SLastBlockReader; typedef struct SFilesetIter { @@ -234,7 +236,6 @@ static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdb static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions, const char* idstr, int8_t* pLevel); static SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level); -static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader); static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader); static int32_t doBuildDataBlock(STsdbReader* pReader); static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader); @@ -1870,7 +1871,8 @@ static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBloc } TSDBROW row = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - TSDBKEY k = TSDBROW_KEY(&row); + TSDBKEY k = {.version = row.version, .ts = row.pBlockData->aTSKEY[row.iRow]}; + pLastBlockReader->currentKey = k.ts; pScanInfo->lastKeyInStt = k.ts; if (!hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->lastBlockDelIndex, &k, pLastBlockReader->order, pVerRange)) { @@ -2582,10 +2584,9 @@ static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScan return nextRowFromLastBlocks(pLBlockReader, pScanInfo, &pReader->verRange); } -static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader) { - TSDBROW* pRow = &tMergeTreeGetRow((&(pLastBlockReader)->mergeTree)); - return pRow->pBlockData->aTSKEY[pRow->iRow]; -} +//static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader) { +// return pLastBlockReader->currentKey; +//} static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { return pLastBlockReader->mergeTree.pIter != NULL; } From 81e54541dec8e044665bbc6ef04ccdc0d531a0e4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Apr 2023 14:24:24 +0800 Subject: [PATCH 095/109] refactor: do some internal refactor. --- source/dnode/vnode/src/inc/tsdb.h | 2 +- source/dnode/vnode/src/tsdb/tsdbCache.c | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 50 ++++++++++++------------- 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 0de1d7de61..8ce40ac293 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -769,7 +769,7 @@ typedef struct SLDataIter { bool ignoreEarlierTs; } SLDataIter; -#define tMergeTreeGetRow(_t) ((_t)->pIter->rInfo.row) +#define tMergeTreeGetRow(_t) (&((_t)->pIter->rInfo.row)) int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo, bool destroyLoadInfo, const char *idStr, bool strictTimeRange, SLDataIter* pLDataIter); diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 42f332c54a..65fc086f8d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -668,7 +668,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa state->state = SFSLASTNEXTROW_FILESET; goto _next_fileset; } - state->row = tMergeTreeGetRow(&state->mergeTree); + state->row = *tMergeTreeGetRow(&state->mergeTree); *ppRow = &state->row; if (TSDBROW_TS(&state->row) <= state->lastTs) { diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 8e70badbc1..0ad9ae1361 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1870,8 +1870,8 @@ static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBloc return false; } - TSDBROW row = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - TSDBKEY k = {.version = row.version, .ts = row.pBlockData->aTSKEY[row.iRow]}; + TSDBROW* pRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + TSDBKEY k = {.version = pRow->version, .ts = pRow->pBlockData->aTSKEY[pRow->iRow]}; pLastBlockReader->currentKey = k.ts; pScanInfo->lastKeyInStt = k.ts; @@ -2029,12 +2029,12 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* } if (minKey == tsLast) { - TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + TSDBROW* fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tsdbRowMergerAdd(&pReader->status.merger, &fRow1, NULL); + tsdbRowMergerAdd(&pReader->status.merger, fRow1, NULL); } else { init = true; - int32_t code = tsdbRowMergerAdd(&pReader->status.merger, &fRow1, pReader->pSchema); + int32_t code = tsdbRowMergerAdd(&pReader->status.merger, fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2081,12 +2081,12 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* } if (minKey == tsLast) { - TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + TSDBROW* fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tsdbRowMergerAdd(pMerger, &fRow1, NULL); + tsdbRowMergerAdd(pMerger, fRow1, NULL); } else { init = true; - int32_t code = tsdbRowMergerAdd(&pReader->status.merger, &fRow1, pReader->pSchema); + int32_t code = tsdbRowMergerAdd(&pReader->status.merger, fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2130,12 +2130,12 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, bool copied = false; int32_t code = TSDB_CODE_SUCCESS; SRow* pTSRow = NULL; - TSDBROW fRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", %s", fRow.pBlockData, fRow.iRow, pLastBlockReader->uid, pReader->idStr); + TSDBROW *pRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", %s", pRow->pBlockData, pRow->iRow, pLastBlockReader->uid, pReader->idStr); // only last block exists if ((!mergeBlockData) || (tsLastBlock != pBlockData->aTSKEY[pDumpInfo->rowIndex])) { - code = tryCopyDistinctRowFromSttBlock(&fRow, pLastBlockReader, pBlockScanInfo, tsLastBlock, pReader, &copied); + code = tryCopyDistinctRowFromSttBlock(pRow, pLastBlockReader, pBlockScanInfo, tsLastBlock, pReader, &copied); if (code) { return code; } @@ -2144,13 +2144,13 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, pBlockScanInfo->lastKey = tsLastBlock; return TSDB_CODE_SUCCESS; } else { - code = tsdbRowMergerAdd(&pReader->status.merger, &fRow, pReader->pSchema); + code = tsdbRowMergerAdd(&pReader->status.merger, pRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } - TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tsdbRowMergerAdd(pMerger, &fRow1, NULL); + TSDBROW* pRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + tsdbRowMergerAdd(pMerger, pRow1, NULL); doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, pMerger, &pReader->verRange, pReader->idStr); code = tsdbRowMergerGetRow(pMerger, &pTSRow); @@ -2168,7 +2168,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, } } } else { // not merge block data - code = tsdbRowMergerAdd(&pReader->status.merger, &fRow, pReader->pSchema); + code = tsdbRowMergerAdd(&pReader->status.merger, pRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2227,8 +2227,8 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); - TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tsdbRowMergerAdd(pMerger, &fRow1, NULL); + TSDBROW* pRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + tsdbRowMergerAdd(pMerger, pRow1, NULL); doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, ts, pMerger, &pReader->verRange, pReader->idStr); @@ -2337,12 +2337,12 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } if (minKey == tsLast) { - TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + TSDBROW* pRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tsdbRowMergerAdd(pMerger, &fRow1, NULL); + tsdbRowMergerAdd(pMerger, pRow1, NULL); } else { init = true; - code = tsdbRowMergerAdd(pMerger, &fRow1, pReader->pSchema); + code = tsdbRowMergerAdd(pMerger, pRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2414,12 +2414,12 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } if (minKey == tsLast) { - TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + TSDBROW* pRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tsdbRowMergerAdd(pMerger, &fRow1, NULL); + tsdbRowMergerAdd(pMerger, pRow1, NULL); } else { init = true; - code = tsdbRowMergerAdd(pMerger, &fRow1, pReader->pSchema); + code = tsdbRowMergerAdd(pMerger, pRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3827,8 +3827,8 @@ int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockSc while (nextRowFromLastBlocks(pLastBlockReader, pScanInfo, pVerRange)) { int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); if (next1 == ts) { - TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tsdbRowMergerAdd(pMerger, &fRow1, NULL); + TSDBROW* pRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + tsdbRowMergerAdd(pMerger, pRow1, NULL); } else { tsdbTrace("uid:%" PRIu64 " last del index:%d, del range:%d, lastKeyInStt:%" PRId64 ", %s", pScanInfo->uid, pScanInfo->lastBlockDelIndex, (int32_t)taosArrayGetSize(pScanInfo->delSkyline), pScanInfo->lastKeyInStt, From 0b6e531dd38d2e25a673f906220cc822faff5021 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Apr 2023 14:56:20 +0800 Subject: [PATCH 096/109] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 55 ++++++++++++-------------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 0ad9ae1361..a2313b40f4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -221,7 +221,7 @@ static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, static int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, int32_t rowIndex); static void setComposedBlockFlag(STsdbReader* pReader, bool composed); -static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order, +static bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t ver, int32_t order, SVersionRange* pVerRange); static int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, @@ -1871,11 +1871,13 @@ static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBloc } TSDBROW* pRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - TSDBKEY k = {.version = pRow->version, .ts = pRow->pBlockData->aTSKEY[pRow->iRow]}; - pLastBlockReader->currentKey = k.ts; - pScanInfo->lastKeyInStt = k.ts; + int64_t key = pRow->pBlockData->aTSKEY[pRow->iRow]; + int64_t ver = pRow->version; - if (!hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->lastBlockDelIndex, &k, pLastBlockReader->order, pVerRange)) { + pLastBlockReader->currentKey = key; + pScanInfo->lastKeyInStt = ver; + + if (!hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->lastBlockDelIndex, key, ver, pLastBlockReader->order, pVerRange)) { // the qualifed ts may equal to k.ts, only a greater version one. // here we need to fallback one step. return true; @@ -2543,8 +2545,7 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDum return false; } - TSDBKEY k = {.ts = ts, .version = ver}; - if (hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->fileDelIndex, &k, pReader->order, + if (hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->fileDelIndex, ts, ver, pReader->order, &pReader->verRange)) { return false; } @@ -2584,10 +2585,6 @@ static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScan return nextRowFromLastBlocks(pLBlockReader, pScanInfo, &pReader->verRange); } -//static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader) { -// return pLastBlockReader->currentKey; -//} - static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { return pLastBlockReader->mergeTree.pIter != NULL; } bool hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* pDumpInfo) { @@ -3562,7 +3559,7 @@ SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_ return (SVersionRange){.minVer = startVer, .maxVer = endVer}; } -bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order, SVersionRange* pVerRange) { +bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t ver, int32_t order, SVersionRange* pVerRange) { if (pDelList == NULL) { return false; } @@ -3574,29 +3571,29 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32 if (asc) { if (*index >= num - 1) { TSDBKEY* last = taosArrayGetLast(pDelList); - ASSERT(pKey->ts >= last->ts); + ASSERT(key >= last->ts); - if (pKey->ts > last->ts) { + if (key > last->ts) { return false; - } else if (pKey->ts == last->ts) { + } else if (key == last->ts) { TSDBKEY* prev = taosArrayGet(pDelList, num - 2); - return (prev->version >= pKey->version && prev->version <= pVerRange->maxVer && + return (prev->version >= ver && prev->version <= pVerRange->maxVer && prev->version >= pVerRange->minVer); } } else { TSDBKEY* pCurrent = taosArrayGet(pDelList, *index); TSDBKEY* pNext = taosArrayGet(pDelList, (*index) + 1); - if (pKey->ts < pCurrent->ts) { + if (key < pCurrent->ts) { return false; } - if (pCurrent->ts <= pKey->ts && pNext->ts >= pKey->ts && pCurrent->version >= pKey->version && + if (pCurrent->ts <= key && pNext->ts >= key && pCurrent->version >= ver && pVerRange->maxVer >= pCurrent->version) { return true; } - while (pNext->ts <= pKey->ts && (*index) < num - 1) { + while (pNext->ts <= key && (*index) < num - 1) { (*index) += 1; if ((*index) < num - 1) { @@ -3608,7 +3605,7 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32 continue; } - if (pCurrent->ts <= pKey->ts && pNext->ts >= pKey->ts && pCurrent->version >= pKey->version && + if (pCurrent->ts <= key && pNext->ts >= key && pCurrent->version >= ver && pVerRange->maxVer >= pCurrent->version) { return true; } @@ -3621,10 +3618,10 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32 if (*index <= 0) { TSDBKEY* pFirst = taosArrayGet(pDelList, 0); - if (pKey->ts < pFirst->ts) { + if (key < pFirst->ts) { return false; - } else if (pKey->ts == pFirst->ts) { - return pFirst->version >= pKey->version; + } else if (key == pFirst->ts) { + return pFirst->version >= ver; } else { ASSERT(0); } @@ -3632,15 +3629,15 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32 TSDBKEY* pCurrent = taosArrayGet(pDelList, *index); TSDBKEY* pPrev = taosArrayGet(pDelList, (*index) - 1); - if (pKey->ts > pCurrent->ts) { + if (key > pCurrent->ts) { return false; } - if (pPrev->ts <= pKey->ts && pCurrent->ts >= pKey->ts && pPrev->version >= pKey->version) { + if (pPrev->ts <= key && pCurrent->ts >= key && pPrev->version >= ver) { return true; } - while (pPrev->ts >= pKey->ts && (*index) > 1) { + while (pPrev->ts >= key && (*index) > 1) { (*index) += step; if ((*index) >= 1) { @@ -3652,7 +3649,7 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32 continue; } - if (pPrev->ts <= pKey->ts && pCurrent->ts >= pKey->ts && pPrev->version >= pKey->version) { + if (pPrev->ts <= key && pCurrent->ts >= key && pPrev->version >= ver) { return true; } } @@ -3680,7 +3677,7 @@ TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* p // it is a valid data version if ((key.version <= pReader->verRange.maxVer && key.version >= pReader->verRange.minVer) && - (!hasBeenDropped(pDelList, &pIter->index, &key, pReader->order, &pReader->verRange))) { + (!hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, pReader->order, &pReader->verRange))) { return pRow; } @@ -3699,7 +3696,7 @@ TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* p } if (key.version <= pReader->verRange.maxVer && key.version >= pReader->verRange.minVer && - (!hasBeenDropped(pDelList, &pIter->index, &key, pReader->order, &pReader->verRange))) { + (!hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, pReader->order, &pReader->verRange))) { return pRow; } } From 8dbc3b34572c71b2d939df5996b76a6da0a51cf9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Apr 2023 15:07:37 +0800 Subject: [PATCH 097/109] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index a2313b40f4..b54ade12e5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -161,6 +161,7 @@ typedef struct SReaderStatus { SDataBlockIter blockIter; SLDataIter* pLDataIter; SRowMerger merger; + SColumnInfoData* pPrimaryTsCol; // primary time stamp output col info data } SReaderStatus; typedef struct SBlockInfoBuf { @@ -780,6 +781,8 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd pSup->tsColAgg.colId = PRIMARYKEY_TIMESTAMP_COL_ID; + setColumnIdSlotList(pSup, pCond->colList, pCond->pSlotList, pCond->numOfCols); + code = tBlockDataCreate(&pReader->status.fileBlockData); if (code != TSDB_CODE_SUCCESS) { terrno = code; @@ -792,7 +795,8 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd goto _end; } - setColumnIdSlotList(pSup, pCond->colList, pCond->pSlotList, pCond->numOfCols); + ASSERT (pReader->suppInfo.colId[0] == PRIMARYKEY_TIMESTAMP_COL_ID); + pReader->status.pPrimaryTsCol = taosArrayGet(pResBlock->pDataBlock, pSup->slotId[0]); tsdbInitReaderLock(pReader); @@ -4091,11 +4095,12 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S int32_t code = TSDB_CODE_SUCCESS; SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; - if (pReader->suppInfo.colId[i] == PRIMARYKEY_TIMESTAMP_COL_ID) { - SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); - ((int64_t*)pColData->pData)[outputRowIndex] = pBlockData->aTSKEY[rowIndex]; +// ASSERT (pReader->suppInfo.colId[i] == PRIMARYKEY_TIMESTAMP_COL_ID);// { +// SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); +// ((int64_t*)pColData->pData)[outputRowIndex] = pBlockData->aTSKEY[rowIndex]; + ((int64_t*)pReader->status.pPrimaryTsCol->pData)[outputRowIndex] = pBlockData->aTSKEY[rowIndex]; i += 1; - } +// } SColVal cv = {0}; int32_t numOfInputCols = pBlockData->nColData; From bca5fc7893e8f6e539ef28a3f433594c0f987a49 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Apr 2023 15:28:54 +0800 Subject: [PATCH 098/109] fix(query): fix memory leak. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index b54ade12e5..03616df21b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -4478,6 +4478,7 @@ void tsdbReaderClose(STsdbReader* pReader) { tsdbUninitReaderLock(pReader); + taosMemoryFreeClear(pReader->status.pLDataIter); taosMemoryFree(pReader->status.uidList.tableUidList); SIOCostSummary* pCost = &pReader->cost; From 16fe48f770f6b8a101d36acb8fe6eb94e4bcb8fb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Apr 2023 16:09:58 +0800 Subject: [PATCH 099/109] fix(query): fix memory leak. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 03616df21b..0ad5a44919 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3378,14 +3378,16 @@ static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBl SBlockNumber num = {0}; SArray* pTableList = taosArrayInit(40, POINTER_BYTES); - int32_t code = moveToNextFile(pReader, &num, pTableList); + int32_t code = moveToNextFile(pReader, &num, pTableList); if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(pTableList); return code; } // all data files are consumed, try data in buffer if (num.numOfBlocks + num.numOfLastFiles == 0) { pReader->status.loadFromFile = false; + taosArrayDestroy(pTableList); return code; } @@ -3400,6 +3402,7 @@ static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBl // set the correct start position according to the query time window initBlockDumpInfo(pReader, pBlockIter); + taosArrayDestroy(pTableList); return code; } From aede1090ff2c6f7144de1d88f647bcda18174178 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 25 Apr 2023 16:23:30 +0800 Subject: [PATCH 100/109] fix: the latest schema may delete columns --- source/dnode/vnode/src/tsdb/tsdbUtil.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 8e778da877..8cbbcb751c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1218,9 +1218,10 @@ int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, int32_t iColumn = 1; STColumn *pTColumn = &pTSchema->columns[iColumn]; for (int32_t iCid = 0; iCid < nCid; iCid++) { - if (ASSERTS(pTColumn != NULL, "invalid input param")) { - code = TSDB_CODE_INVALID_PARA; - goto _exit; + + // aCid array (from taos client catalog) contains columns that does not exist in the pTSchema. the pTSchema is newer + if (pTColumn == NULL) { + continue; } while (pTColumn->colId < aCid[iCid]) { @@ -1229,9 +1230,8 @@ int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, pTColumn = &pTSchema->columns[iColumn]; } - if (ASSERTS(pTColumn->colId == aCid[iCid], "invalid input param")) { - code = TSDB_CODE_INVALID_PARA; - goto _exit; + if (pTColumn->colId != aCid[iCid]) { + continue; } tColDataInit(&pBlockData->aColData[iCid], pTColumn->colId, pTColumn->type, From ee93500619e91ff6e7851171c2b38ec9d13859f8 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 25 Apr 2023 17:03:58 +0800 Subject: [PATCH 101/109] fix: log file size over 4G report error on windows --- source/os/src/osFile.c | 3 ++- source/util/src/tlog.c | 4 +--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index aab547223f..9833e3ba45 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -538,10 +538,11 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { return -1; } - struct stat fileStat; #ifdef WINDOWS + struct __stat64 fileStat; int32_t code = _fstat(pFile->fd, &fileStat); #else + struct stat fileStat; int32_t code = fstat(pFile->fd, &fileStat); #endif if (code < 0) { diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 2a18f420cd..d415379f92 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -347,7 +347,6 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { char name[LOG_FILE_NAME_LEN + 50] = "\0"; int32_t logstat0_mtime, logstat1_mtime; - int32_t size; tsLogObj.maxLines = maxLines; tsLogObj.fileNum = maxFileNum; @@ -395,8 +394,7 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { printf("\nfailed to fstat log file:%s, reason:%s\n", fileName, strerror(errno)); return -1; } - size = (int32_t)filesize; - tsLogObj.lines = size / 60; + tsLogObj.lines = (int32_t)(filesize / 60); taosLSeekFile(tsLogObj.logHandle->pFile, 0, SEEK_END); From 97a5ebac1c5154458bf13a5e7b63a41cd555713d Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 25 Apr 2023 17:10:12 +0800 Subject: [PATCH 102/109] fix: miss out _fstat64 --- source/os/src/osFile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 9833e3ba45..dd670595f0 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -540,7 +540,7 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { #ifdef WINDOWS struct __stat64 fileStat; - int32_t code = _fstat(pFile->fd, &fileStat); + int32_t code = _fstat64(pFile->fd, &fileStat); #else struct stat fileStat; int32_t code = fstat(pFile->fd, &fileStat); From 49728a235e83f58df0a995e6a4c88779d9980d9b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 25 Apr 2023 17:32:45 +0800 Subject: [PATCH 103/109] fix: more fix --- source/dnode/vnode/src/inc/tsdb.h | 5 +- source/dnode/vnode/src/tsdb/tsdbDataIter.c | 61 ++++++++++++++-------- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 26 +++++---- 3 files changed, 57 insertions(+), 35 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 2a85b191a4..4d3111a9b7 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -322,8 +322,9 @@ int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData, #define TSDB_STT_FILE_DATA_ITER 2 #define TSDB_TOMB_FILE_DATA_ITER 3 -#define TSDB_FILTER_FLAG_BY_VERSION 0x1 -#define TSDB_FILTER_FLAG_BY_TABLEID 0x2 +#define TSDB_FILTER_FLAG_BY_VERSION 0x1 +#define TSDB_FILTER_FLAG_BY_TABLEID 0x2 +#define TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE 0x4 #define TSDB_RBTN_TO_DATA_ITER(pNode) ((STsdbDataIter2 *)(((char *)pNode) - offsetof(STsdbDataIter2, rbtn))) /* open */ diff --git a/source/dnode/vnode/src/tsdb/tsdbDataIter.c b/source/dnode/vnode/src/tsdb/tsdbDataIter.c index 3299a2f497..e27aec5b1b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataIter.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataIter.c @@ -14,6 +14,7 @@ */ #include "tsdb.h" +#include "vnodeInt.h" // STsdbDataIter2 /* open */ @@ -202,13 +203,6 @@ static int32_t tsdbDataFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo* for (;;) { while (pIter->dIter.iRow < pIter->dIter.bData.nRow) { if (pFilterInfo) { - if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_TABLEID) { - if (pFilterInfo->tbid.uid == pIter->dIter.bData.uid) { - pIter->dIter.iRow = pIter->dIter.bData.nRow; - continue; - } - } - if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_VERSION) { if (pIter->dIter.bData.aVersion[pIter->dIter.iRow] < pFilterInfo->sver || pIter->dIter.bData.aVersion[pIter->dIter.iRow] > pFilterInfo->ever) { @@ -232,13 +226,6 @@ static int32_t tsdbDataFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo* // filter if (pFilterInfo) { - if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_TABLEID) { - if (tTABLEIDCmprFn(&pFilterInfo->tbid, &pIter->rowInfo) == 0) { - pIter->dIter.iDataBlk = pIter->dIter.mDataBlk.nItem; - continue; - } - } - if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_VERSION) { if (pFilterInfo->sver > dataBlk.maxVer || pFilterInfo->ever < dataBlk.minVer) { pIter->dIter.iDataBlk++; @@ -262,13 +249,23 @@ static int32_t tsdbDataFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo* if (pIter->dIter.iBlockIdx < taosArrayGetSize(pIter->dIter.aBlockIdx)) { SBlockIdx* pBlockIdx = taosArrayGet(pIter->dIter.aBlockIdx, pIter->dIter.iBlockIdx); - if (pFilterInfo && (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_TABLEID)) { - int32_t c = tTABLEIDCmprFn(pBlockIdx, &pFilterInfo->tbid); - if (c == 0) { - pIter->dIter.iBlockIdx++; - continue; - } else if (c < 0) { - ASSERT(0); + if (pFilterInfo) { + if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_TABLEID) { + int32_t c = tTABLEIDCmprFn(pBlockIdx, &pFilterInfo->tbid); + if (c == 0) { + pIter->dIter.iBlockIdx++; + continue; + } else if (c < 0) { + ASSERT(0); + } + } + + if (pFilterInfo->flag & TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE) { + SMetaInfo info; + if (metaGetInfo(pIter->dIter.pReader->pTsdb->pVnode->pMeta, pBlockIdx->uid, &info, NULL)) { + pIter->dIter.iBlockIdx++; + continue; + } } } @@ -304,14 +301,24 @@ static int32_t tsdbSttFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo* p for (;;) { while (pIter->sIter.iRow < pIter->sIter.bData.nRow) { if (pFilterInfo) { + int64_t uid = pIter->sIter.bData.uid ? pIter->sIter.bData.uid : pIter->sIter.bData.aUid[pIter->sIter.iRow]; if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_TABLEID) { - int64_t uid = pIter->sIter.bData.uid ? pIter->sIter.bData.uid : pIter->sIter.bData.aUid[pIter->sIter.iRow]; if (pFilterInfo->tbid.uid == uid) { pIter->sIter.iRow++; continue; } } + if (pFilterInfo->flag & TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE) { + if (pIter->rowInfo.uid != uid) { + SMetaInfo info; + if (metaGetInfo(pIter->sIter.pReader->pTsdb->pVnode->pMeta, uid, &info, NULL)) { + pIter->sIter.iRow++; + continue; + } + } + } + if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_VERSION) { if (pFilterInfo->sver > pIter->sIter.bData.aVersion[pIter->sIter.iRow] || pFilterInfo->ever < pIter->sIter.bData.aVersion[pIter->sIter.iRow]) { @@ -395,6 +402,16 @@ static int32_t tsdbTombFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo* if (pIter->tIter.iDelIdx < taosArrayGetSize(pIter->tIter.aDelIdx)) { SDelIdx* pDelIdx = taosArrayGet(pIter->tIter.aDelIdx, pIter->tIter.iDelIdx); + if (pFilterInfo) { + if (pFilterInfo->flag & TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE) { + SMetaInfo info; + if (metaGetInfo(pIter->dIter.pReader->pTsdb->pVnode->pMeta, pDelIdx->uid, &info, NULL)) { + pIter->tIter.iDelIdx++; + continue; + } + } + } + code = tsdbReadDelData(pIter->tIter.pReader, pDelIdx, pIter->tIter.aDelData); TSDB_CHECK_CODE(code, lino, _exit); diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index aed863d194..dfea125cc1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -70,10 +70,11 @@ static int32_t tsdbSnapReadFileDataStart(STsdbSnapReader* pReader) { if (pReader->pIter) { // iter to next with filter info (sver, ever) - code = tsdbDataIterNext2(pReader->pIter, - &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION, // flag - .sver = pReader->sver, - .ever = pReader->ever}); + code = tsdbDataIterNext2( + pReader->pIter, + &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION | TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE, // flag + .sver = pReader->sver, + .ever = pReader->ever}); TSDB_CHECK_CODE(code, lino, _exit); if (pReader->pIter->rowInfo.suid || pReader->pIter->rowInfo.uid) { @@ -94,10 +95,11 @@ static int32_t tsdbSnapReadFileDataStart(STsdbSnapReader* pReader) { if (pReader->pIter) { // iter to valid row - code = tsdbDataIterNext2(pReader->pIter, - &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION, // flag - .sver = pReader->sver, - .ever = pReader->ever}); + code = tsdbDataIterNext2( + pReader->pIter, + &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION | TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE, // flag + .sver = pReader->sver, + .ever = pReader->ever}); TSDB_CHECK_CODE(code, lino, _exit); if (pReader->pIter->rowInfo.suid || pReader->pIter->rowInfo.uid) { @@ -139,7 +141,8 @@ static int32_t tsdbSnapReadNextRow(STsdbSnapReader* pReader, SRowInfo** ppRowInf int32_t lino = 0; if (pReader->pIter) { - code = tsdbDataIterNext2(pReader->pIter, &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION, // flag + code = tsdbDataIterNext2(pReader->pIter, &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION | + TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE, // flag .sver = pReader->sver, .ever = pReader->ever}); TSDB_CHECK_CODE(code, lino, _exit); @@ -346,8 +349,9 @@ static int32_t tsdbSnapReadNextTombData(STsdbSnapReader* pReader, SDelInfo** ppD int32_t lino = 0; code = tsdbDataIterNext2( - pReader->pTIter, - &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION, .sver = pReader->sver, .ever = pReader->ever}); + pReader->pTIter, &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION | TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE, + .sver = pReader->sver, + .ever = pReader->ever}); TSDB_CHECK_CODE(code, lino, _exit); if (ppDelInfo) { From e584bb3ea17e272135a5cbfdb8b13b94749d991c Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 25 Apr 2023 17:39:28 +0800 Subject: [PATCH 104/109] feat: subtable level privilege --- include/libs/nodes/querynodes.h | 2 + source/client/src/clientMain.c | 32 +++--- source/libs/catalog/src/ctgUtil.c | 6 +- source/libs/nodes/src/nodesUtilFuncs.c | 2 + source/libs/parser/src/parAuthenticator.c | 11 +- source/libs/parser/src/parInsertSql.c | 114 ++++++++++++++++++--- source/libs/scalar/src/scalar.c | 26 ++--- tests/parallel_test/cases.task | 1 + tests/script/tsim/user/privilege_table.sim | 34 ++++-- tests/script/win-test-file | 1 + 10 files changed, 177 insertions(+), 52 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 480912a8cf..9569cfe055 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -379,6 +379,8 @@ typedef struct SVnodeModifyOpStmt { SName usingTableName; const char* pBoundCols; struct STableMeta* pTableMeta; + SNode* pTagCond; + SArray* pTableTag; SHashObj* pVgroupsHashObj; SHashObj* pTableBlockHashObj; // SHashObj SHashObj* pSubTableHashObj; diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 54ae1ab4b3..2ebc8e7379 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -108,7 +108,7 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha if (pass == NULL) { pass = TSDB_DEFAULT_PASS; } - + STscObj *pObj = taos_connect_internal(ip, user, pass, NULL, db, port, CONN_TYPE__QUERY); if (pObj) { int64_t *rid = taosMemoryCalloc(1, sizeof(int64_t)); @@ -359,11 +359,11 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) case TSDB_DATA_TYPE_NCHAR: { int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE); if (fields[i].type == TSDB_DATA_TYPE_BINARY) { - if(ASSERT(charLen <= fields[i].bytes && charLen >= 0)){ + if (ASSERT(charLen <= fields[i].bytes && charLen >= 0)) { tscError("taos_print_row error binary. charLen:%d, fields[i].bytes:%d", charLen, fields[i].bytes); } } else { - if(ASSERT(charLen <= fields[i].bytes * TSDB_NCHAR_SIZE && charLen >= 0)){ + if (ASSERT(charLen <= fields[i].bytes * TSDB_NCHAR_SIZE && charLen >= 0)) { tscError("taos_print_row error. charLen:%d, fields[i].bytes:%d", charLen, fields[i].bytes); } } @@ -705,16 +705,16 @@ int taos_get_current_db(TAOS *taos, char *database, int len, int *required) { int code = TSDB_CODE_SUCCESS; taosThreadMutexLock(&pTscObj->mutex); - if(database == NULL || len <= 0){ - if(required != NULL) *required = strlen(pTscObj->db) + 1; + if (database == NULL || len <= 0) { + if (required != NULL) *required = strlen(pTscObj->db) + 1; terrno = TSDB_CODE_INVALID_PARA; code = -1; - }else if(len < strlen(pTscObj->db) + 1){ + } else if (len < strlen(pTscObj->db) + 1) { tstrncpy(database, pTscObj->db, len); - if(required) *required = strlen(pTscObj->db) + 1; + if (required) *required = strlen(pTscObj->db) + 1; terrno = TSDB_CODE_INVALID_PARA; code = -1; - }else{ + } else { strcpy(database, pTscObj->db); code = 0; } @@ -741,6 +741,7 @@ static void destoryCatalogReq(SCatalogReq *pCatalogReq) { taosArrayDestroy(pCatalogReq->pUser); taosArrayDestroy(pCatalogReq->pTableIndex); taosArrayDestroy(pCatalogReq->pTableCfg); + taosArrayDestroy(pCatalogReq->pTableTag); taosMemoryFree(pCatalogReq); } @@ -976,7 +977,7 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { if (TSDB_CODE_SUCCESS == code) { pRequest->stmtType = pRequest->pQuery->pRoot->type; code = phaseAsyncQuery(pWrapper); - } + } if (TSDB_CODE_SUCCESS != code) { tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code), @@ -1044,11 +1045,11 @@ static void fetchCallback(void *pResult, void *param, int32_t code) { } void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { - if(ASSERT(res != NULL && fp != NULL)){ + if (ASSERT(res != NULL && fp != NULL)) { tscError("taos_fetch_rows_a invalid paras"); return; } - if(ASSERT(TD_RES_QUERY(res))){ + if (ASSERT(TD_RES_QUERY(res))) { tscError("taos_fetch_rows_a res is NULL"); return; } @@ -1094,11 +1095,11 @@ void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { } void taos_fetch_raw_block_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { - if(ASSERT(res != NULL && fp != NULL)){ + if (ASSERT(res != NULL && fp != NULL)) { tscError("taos_fetch_rows_a invalid paras"); return; } - if(ASSERT(TD_RES_QUERY(res))){ + if (ASSERT(TD_RES_QUERY(res))) { tscError("taos_fetch_rows_a res is NULL"); return; } @@ -1113,11 +1114,11 @@ void taos_fetch_raw_block_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { } const void *taos_get_raw_block(TAOS_RES *res) { - if(ASSERT(res != NULL)){ + if (ASSERT(res != NULL)) { tscError("taos_fetch_rows_a invalid paras"); return NULL; } - if(ASSERT(TD_RES_QUERY(res))){ + if (ASSERT(TD_RES_QUERY(res))) { tscError("taos_fetch_rows_a res is NULL"); return NULL; } @@ -1275,7 +1276,6 @@ _return: return code; } - int taos_load_table_info(TAOS *taos, const char *tableNameList) { if (NULL == taos) { terrno = TSDB_CODE_TSC_DISCONNECTED; diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index a42a189e75..9274c1ef92 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -170,6 +170,9 @@ void ctgFreeSMetaData(SMetaData* pData) { taosArrayDestroy(pData->pTableCfg); pData->pTableCfg = NULL; + taosArrayDestroy(pData->pTableTag); + pData->pTableTag = NULL; + taosMemoryFreeClear(pData->pSvrVer); } @@ -530,7 +533,7 @@ void ctgFreeTaskRes(CTG_TASK_TYPE type, void** pRes) { } case CTG_TASK_GET_USER: { if (*pRes) { - SUserAuthRes* pAuth = (SUserAuthRes *)*pRes; + SUserAuthRes* pAuth = (SUserAuthRes*)*pRes; nodesDestroyNode(pAuth->pCond); taosMemoryFreeClear(*pRes); } @@ -711,6 +714,7 @@ void ctgFreeTaskCtx(SCtgTask* pTask) { SCtgTbTagCtx* taskCtx = (SCtgTbTagCtx*)pTask->taskCtx; taosMemoryFreeClear(taskCtx->pName); taosMemoryFreeClear(taskCtx->pVgInfo); + taosMemoryFreeClear(taskCtx); break; } case CTG_TASK_GET_DB_VGROUP: diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 422a196c50..f71eef7969 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -827,6 +827,8 @@ void nodesDestroyNode(SNode* pNode) { SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pNode; destroyVgDataBlockArray(pStmt->pDataBlocks); taosMemoryFreeClear(pStmt->pTableMeta); + nodesDestroyNode(pStmt->pTagCond); + taosArrayDestroy(pStmt->pTableTag); taosHashCleanup(pStmt->pVgroupsHashObj); taosHashCleanup(pStmt->pSubTableHashObj); taosHashCleanup(pStmt->pTableNameHashObj); diff --git a/source/libs/parser/src/parAuthenticator.c b/source/libs/parser/src/parAuthenticator.c index d182f5bd73..1586d8128b 100644 --- a/source/libs/parser/src/parAuthenticator.c +++ b/source/libs/parser/src/parAuthenticator.c @@ -91,17 +91,22 @@ static int32_t mergeStableTagCond(SNode** pWhere, SNode* pTagCond) { } static int32_t appendStableTagCond(SNode** pWhere, SNode* pTagCond) { + SNode* pTagCondCopy = nodesCloneNode(pTagCond); + if (NULL == pTagCondCopy) { + return TSDB_CODE_OUT_OF_MEMORY; + } + if (NULL == *pWhere) { - *pWhere = pTagCond; + *pWhere = pTagCondCopy; return TSDB_CODE_SUCCESS; } if (QUERY_NODE_LOGIC_CONDITION == nodeType(*pWhere) && LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)*pWhere)->condType) { - return nodesListStrictAppend(((SLogicConditionNode*)*pWhere)->pParameterList, pTagCond); + return nodesListStrictAppend(((SLogicConditionNode*)*pWhere)->pParameterList, pTagCondCopy); } - return mergeStableTagCond(pWhere, pTagCond); + return mergeStableTagCond(pWhere, pTagCondCopy); } static EDealRes authSelectImpl(SNode* pNode, void* pContext) { diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index eb2efd573d..02de9f227d 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -53,6 +53,7 @@ typedef struct SInsertParseContext { bool missCache; bool usingDuplicateTable; bool forceUpdate; + bool needTableTagVal; } SInsertParseContext; typedef int32_t (*_row_append_fn_t)(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param); @@ -577,28 +578,39 @@ static int32_t rewriteTagCondColumnImpl(STagVal* pVal, SNode** pNode) { if (NULL == pValue) { return TSDB_CODE_OUT_OF_MEMORY; } - pValue->node.resType.type = pVal->type; + + pValue->node.resType = ((SColumnNode*)*pNode)->node.resType; + nodesDestroyNode(*pNode); + *pNode = (SNode*)pValue; + switch (pVal->type) { case TSDB_DATA_TYPE_BOOL: pValue->datum.b = *(int8_t*)(&pVal->i64); + *(bool*)&pValue->typeData = pValue->datum.b; break; case TSDB_DATA_TYPE_TINYINT: pValue->datum.i = *(int8_t*)(&pVal->i64); + *(int8_t*)&pValue->typeData = pValue->datum.i; break; case TSDB_DATA_TYPE_SMALLINT: pValue->datum.i = *(int16_t*)(&pVal->i64); + *(int16_t*)&pValue->typeData = pValue->datum.i; break; case TSDB_DATA_TYPE_INT: pValue->datum.i = *(int32_t*)(&pVal->i64); + *(int32_t*)&pValue->typeData = pValue->datum.i; break; case TSDB_DATA_TYPE_BIGINT: pValue->datum.i = pVal->i64; + pValue->typeData = pValue->datum.i; break; case TSDB_DATA_TYPE_FLOAT: pValue->datum.d = *(float*)(&pVal->i64); + *(float*)&pValue->typeData = pValue->datum.d; break; case TSDB_DATA_TYPE_DOUBLE: pValue->datum.d = *(double*)(&pVal->i64); + *(double*)&pValue->typeData = pValue->datum.d; break; case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_NCHAR: @@ -611,18 +623,23 @@ static int32_t rewriteTagCondColumnImpl(STagVal* pVal, SNode** pNode) { break; case TSDB_DATA_TYPE_TIMESTAMP: pValue->datum.i = pVal->i64; + pValue->typeData = pValue->datum.i; break; case TSDB_DATA_TYPE_UTINYINT: pValue->datum.i = *(uint8_t*)(&pVal->i64); + *(uint8_t*)&pValue->typeData = pValue->datum.i; break; case TSDB_DATA_TYPE_USMALLINT: pValue->datum.i = *(uint16_t*)(&pVal->i64); + *(uint16_t*)&pValue->typeData = pValue->datum.i; break; case TSDB_DATA_TYPE_UINT: pValue->datum.i = *(uint32_t*)(&pVal->i64); + *(uint32_t*)&pValue->typeData = pValue->datum.i; break; case TSDB_DATA_TYPE_UBIGINT: pValue->datum.i = *(uint64_t*)(&pVal->i64); + *(uint64_t*)&pValue->typeData = pValue->datum.i; break; case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_VARBINARY: @@ -667,16 +684,15 @@ static int32_t checkTagCondResult(SNode* pResult) { : TSDB_CODE_PAR_PERMISSION_DENIED; } -int32_t checkSubtablePrivilege(SArray* pTagVals, SArray* pTagName, SNode* pCond) { - int32_t code = setTagVal(pTagVals, pTagName, pCond); - SNode* pNew = NULL; +static int32_t checkSubtablePrivilege(SArray* pTagVals, SArray* pTagName, SNode** pCond) { + int32_t code = setTagVal(pTagVals, pTagName, *pCond); if (TSDB_CODE_SUCCESS == code) { - code = scalarCalculateConstants(pCond, &pNew); + code = scalarCalculateConstants(*pCond, pCond); } if (TSDB_CODE_SUCCESS == code) { - code = checkTagCondResult(pNew); + code = checkTagCondResult(*pCond); } - nodesDestroyNode(pNew); + NODES_DESTORY_NODE(*pCond); return code; } @@ -716,6 +732,10 @@ static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt } } + if (TSDB_CODE_SUCCESS == code && NULL != pStmt->pTagCond) { + code = checkSubtablePrivilege(pTagVals, pTagName, &pStmt->pTagCond); + } + if (TSDB_CODE_SUCCESS == code && !isParseBindParam && !isJson) { code = tTagNew(pTagVals, 1, false, &pTag); } @@ -843,7 +863,7 @@ static void setUserAuthInfo(SParseContext* pCxt, SName* pTbName, SUserAuthInfo* pInfo->type = AUTH_TYPE_WRITE; } -static int32_t checkAuth(SParseContext* pCxt, SName* pTbName, bool* pMissCache) { +static int32_t checkAuth(SParseContext* pCxt, SName* pTbName, bool* pMissCache, SNode** pTagCond) { int32_t code = TSDB_CODE_SUCCESS; SUserAuthInfo authInfo = {0}; setUserAuthInfo(pCxt, pTbName, &authInfo); @@ -863,11 +883,28 @@ static int32_t checkAuth(SParseContext* pCxt, SName* pTbName, bool* pMissCache) *pMissCache = true; } else if (!authRes.pass) { code = TSDB_CODE_PAR_PERMISSION_DENIED; + } else if (NULL != authRes.pCond) { + *pTagCond = authRes.pCond; } } return code; } +static int32_t checkAuthForTable(SParseContext* pCxt, SName* pTbName, bool* pMissCache, bool* pNeedTableTagVal) { + SNode* pTagCond = NULL; + int32_t code = checkAuth(pCxt, pTbName, pMissCache, &pTagCond); + if (TSDB_CODE_SUCCESS == code) { + *pNeedTableTagVal = ((*pMissCache) || (NULL != pTagCond)); + *pMissCache = (NULL != pTagCond); + } + nodesDestroyNode(pTagCond); + return code; +} + +static int32_t checkAuthForStable(SParseContext* pCxt, SName* pTbName, bool* pMissCache, SNode** pTagCond) { + return checkAuth(pCxt, pTbName, pMissCache, pTagCond); +} + static int32_t getTableMeta(SInsertParseContext* pCxt, SName* pTbName, bool isStb, STableMeta** pTableMeta, bool* pMissCache) { SParseContext* pComCxt = pCxt->pComCxt; @@ -970,7 +1007,7 @@ static int32_t getTargetTableSchema(SInsertParseContext* pCxt, SVnodeModifyOpStm return TSDB_CODE_SUCCESS; } - int32_t code = checkAuth(pCxt->pComCxt, &pStmt->targetTableName, &pCxt->missCache); + int32_t code = checkAuthForTable(pCxt->pComCxt, &pStmt->targetTableName, &pCxt->missCache, &pCxt->needTableTagVal); if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { code = getTableMetaAndVgroup(pCxt, pStmt, &pCxt->missCache); } @@ -993,7 +1030,7 @@ static int32_t getUsingTableSchema(SInsertParseContext* pCxt, SVnodeModifyOpStmt return TSDB_CODE_SUCCESS; } - int32_t code = checkAuth(pCxt->pComCxt, &pStmt->targetTableName, &pCxt->missCache); + int32_t code = checkAuthForStable(pCxt->pComCxt, &pStmt->usingTableName, &pCxt->missCache, &pStmt->pTagCond); if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { code = getTableMeta(pCxt, &pStmt->usingTableName, true, &pStmt->pTableMeta, &pCxt->missCache); } @@ -1606,6 +1643,8 @@ static int32_t parseInsertTableClauseBottom(SInsertParseContext* pCxt, SVnodeMod static void resetEnvPreTable(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { insDestroyBoundColInfo(&pCxt->tags); taosMemoryFreeClear(pStmt->pTableMeta); + nodesDestroyNode(pStmt->pTagCond); + taosArrayDestroy(pStmt->pTableTag); tdDestroySVCreateTbReq(pStmt->pCreateTblReq); taosMemoryFreeClear(pStmt->pCreateTblReq); pCxt->missCache = false; @@ -1780,14 +1819,18 @@ static int32_t createInsertQuery(SInsertParseContext* pCxt, SQuery** pOutput) { return code; } -static int32_t checkAuthFromMetaData(const SArray* pUsers) { +static int32_t checkAuthFromMetaData(const SArray* pUsers, SNode** pTagCond) { if (1 != taosArrayGetSize(pUsers)) { return TSDB_CODE_FAILED; } SMetaRes* pRes = taosArrayGet(pUsers, 0); if (TSDB_CODE_SUCCESS == pRes->code) { - return (*(bool*)pRes->pRes) ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED; + SUserAuthRes* pAuth = pRes->pRes; + if (NULL != pAuth->pCond) { + *pTagCond = nodesCloneNode(pAuth->pCond); + } + return pAuth->pass ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED; } return pRes->code; } @@ -1826,9 +1869,40 @@ static int32_t getTableVgroupFromMetaData(const SArray* pTables, SVnodeModifyOpS sizeof(SVgroupInfo)); } +static int32_t buildTagNameFromMeta(STableMeta* pMeta, SArray** pTagName) { + *pTagName = taosArrayInit(pMeta->tableInfo.numOfTags, TSDB_COL_NAME_LEN); + if (NULL == *pTagName) { + return TSDB_CODE_OUT_OF_MEMORY; + } + SSchema* pSchema = getTableTagSchema(pMeta); + for (int32_t i = 0; i < pMeta->tableInfo.numOfTags; ++i) { + taosArrayPush(*pTagName, pSchema[i].name); + } + return TSDB_CODE_SUCCESS; +} + +static int32_t checkSubtablePrivilegeForTable(const SArray* pTables, SVnodeModifyOpStmt* pStmt) { + if (1 != taosArrayGetSize(pTables)) { + return TSDB_CODE_FAILED; + } + + SMetaRes* pRes = taosArrayGet(pTables, 0); + if (TSDB_CODE_SUCCESS != pRes->code) { + return pRes->code; + } + + SArray* pTagName = NULL; + int32_t code = buildTagNameFromMeta(pStmt->pTableMeta, &pTagName); + if (TSDB_CODE_SUCCESS == code) { + code = checkSubtablePrivilege((SArray*)pRes->pRes, pTagName, &pStmt->pTagCond); + } + taosArrayDestroy(pTagName); + return code; +} + static int32_t getTableSchemaFromMetaData(SInsertParseContext* pCxt, const SMetaData* pMetaData, SVnodeModifyOpStmt* pStmt, bool isStb) { - int32_t code = checkAuthFromMetaData(pMetaData->pUser); + int32_t code = checkAuthFromMetaData(pMetaData->pUser, &pStmt->pTagCond); if (TSDB_CODE_SUCCESS == code) { code = getTableMetaFromMetaData(pMetaData->pTableMeta, &pStmt->pTableMeta); } @@ -1841,6 +1915,9 @@ static int32_t getTableSchemaFromMetaData(SInsertParseContext* pCxt, const SMeta if (TSDB_CODE_SUCCESS == code) { code = getTableVgroupFromMetaData(pMetaData->pTableHash, pStmt, isStb); } + if (TSDB_CODE_SUCCESS == code && !isStb && NULL != pStmt->pTagCond) { + code = checkSubtablePrivilegeForTable(pMetaData->pTableTag, pStmt); + } return code; } @@ -1860,6 +1937,8 @@ static void clearCatalogReq(SCatalogReq* pCatalogReq) { pCatalogReq->pTableHash = NULL; taosArrayDestroy(pCatalogReq->pUser); pCatalogReq->pUser = NULL; + taosArrayDestroy(pCatalogReq->pTableTag); + pCatalogReq->pTableTag = NULL; } static int32_t setVnodeModifOpStmt(SInsertParseContext* pCxt, SCatalogReq* pCatalogReq, const SMetaData* pMetaData, @@ -2033,8 +2112,15 @@ static int32_t buildInsertUserAuthReq(const char* pUser, SName* pName, SArray** return TSDB_CODE_SUCCESS; } +static int32_t buildInsertTableTagReq(SName* pName, SArray** pTables) { return buildInsertTableReq(pName, pTables); } + static int32_t buildInsertCatalogReq(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SCatalogReq* pCatalogReq) { - int32_t code = buildInsertUserAuthReq(pCxt->pComCxt->pUser, &pStmt->targetTableName, &pCatalogReq->pUser); + int32_t code = buildInsertUserAuthReq( + pCxt->pComCxt->pUser, (0 == pStmt->usingTableName.type ? &pStmt->targetTableName : &pStmt->usingTableName), + &pCatalogReq->pUser); + if (TSDB_CODE_SUCCESS == code && pCxt->needTableTagVal) { + code = buildInsertTableTagReq(&pStmt->targetTableName, &pCatalogReq->pTableTag); + } if (TSDB_CODE_SUCCESS == code) { if (0 == pStmt->usingTableName.type) { code = buildInsertDbReq(&pStmt->targetTableName, &pCatalogReq->pTableMeta); diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 0521076d23..d9295656e8 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -53,6 +53,7 @@ int32_t sclCreateColumnInfoData(SDataType *pType, int32_t numOfRows, SScalarPara int32_t code = colInfoDataEnsureCapacity(pColumnData, numOfRows, true); if (code != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_OUT_OF_MEMORY; + colDataDestroy(pColumnData); taosMemoryFree(pColumnData); return terrno; } @@ -1061,17 +1062,20 @@ int32_t sclConvertOpValueNodeTs(SOperatorNode *node, SScalarCtx *ctx) { if (node->pLeft && SCL_IS_VAR_VALUE_NODE(node->pLeft)) { if (node->pRight && (TSDB_DATA_TYPE_TIMESTAMP == ((SExprNode *)node->pRight)->resType.type)) { - SCL_ERR_JRET(sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, node->pRight), (SValueNode*)node->pLeft)); + SCL_ERR_JRET( + sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, node->pRight), (SValueNode *)node->pLeft)); } } else if (node->pRight && SCL_IS_NOTNULL_CONST_NODE(node->pRight)) { if (node->pLeft && (TSDB_DATA_TYPE_TIMESTAMP == ((SExprNode *)node->pLeft)->resType.type)) { if (SCL_IS_VAR_VALUE_NODE(node->pRight)) { - SCL_ERR_JRET(sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, node->pRight), (SValueNode*)node->pRight)); + SCL_ERR_JRET(sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, node->pRight), + (SValueNode *)node->pRight)); } else if (QUERY_NODE_NODE_LIST == node->pRight->type) { - SNode* pNode; - FOREACH(pNode, ((SNodeListNode*)node->pRight)->pNodeList) { + SNode *pNode; + FOREACH(pNode, ((SNodeListNode *)node->pRight)->pNodeList) { if (SCL_IS_VAR_VALUE_NODE(pNode)) { - SCL_ERR_JRET(sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, pNode), (SValueNode*)pNode)); + SCL_ERR_JRET( + sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, pNode), (SValueNode *)pNode)); } } } @@ -1086,8 +1090,6 @@ _return: return DEAL_RES_ERROR; } - - int32_t sclConvertCaseWhenValueNodeTs(SCaseWhenNode *node, SScalarCtx *ctx) { int32_t code = 0; @@ -1096,19 +1098,20 @@ int32_t sclConvertCaseWhenValueNodeTs(SCaseWhenNode *node, SScalarCtx *ctx) { } if (SCL_IS_VAR_VALUE_NODE(node->pCase)) { - SNode* pNode; + SNode *pNode; FOREACH(pNode, node->pWhenThenList) { SExprNode *pExpr = (SExprNode *)((SWhenThenNode *)pNode)->pWhen; if (TSDB_DATA_TYPE_TIMESTAMP == pExpr->resType.type) { - SCL_ERR_JRET(sclConvertToTsValueNode(pExpr->resType.precision, (SValueNode*)node->pCase)); + SCL_ERR_JRET(sclConvertToTsValueNode(pExpr->resType.precision, (SValueNode *)node->pCase)); break; } } } else if (TSDB_DATA_TYPE_TIMESTAMP == ((SExprNode *)node->pCase)->resType.type) { - SNode* pNode; + SNode *pNode; FOREACH(pNode, node->pWhenThenList) { if (SCL_IS_VAR_VALUE_NODE(((SWhenThenNode *)pNode)->pWhen)) { - SCL_ERR_JRET(sclConvertToTsValueNode(((SExprNode *)node->pCase)->resType.precision, (SValueNode*)((SWhenThenNode *)pNode)->pWhen)); + SCL_ERR_JRET(sclConvertToTsValueNode(((SExprNode *)node->pCase)->resType.precision, + (SValueNode *)((SWhenThenNode *)pNode)->pWhen)); } } } @@ -1271,7 +1274,6 @@ EDealRes sclRewriteLogic(SNode **pNode, SScalarCtx *ctx) { return DEAL_RES_CONTINUE; } - EDealRes sclRewriteOperator(SNode **pNode, SScalarCtx *ctx) { SOperatorNode *node = (SOperatorNode *)*pNode; diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index dda4ec3e84..8ea13e741c 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -722,6 +722,7 @@ ,,y,script,./test.sh -f tsim/user/privilege_db.sim ,,y,script,./test.sh -f tsim/user/privilege_sysinfo.sim ,,y,script,./test.sh -f tsim/user/privilege_topic.sim +,,y,script,./test.sh -f tsim/user/privilege_table.sim ,,y,script,./test.sh -f tsim/db/alter_option.sim ,,y,script,./test.sh -f tsim/db/alter_replica_31.sim ,,y,script,./test.sh -f tsim/db/basic1.sim diff --git a/tests/script/tsim/user/privilege_table.sim b/tests/script/tsim/user/privilege_table.sim index 5256cdd21d..05f91ff5b0 100644 --- a/tests/script/tsim/user/privilege_table.sim +++ b/tests/script/tsim/user/privilege_table.sim @@ -96,7 +96,9 @@ sql connect wxy sql reset query cache; sql_error select tbname, * from test.st1; sql insert into test.st1s1 values(now, 10); +sql insert into test.st1s3 using test.st1 tags(1, 'dachang') values(now, 100); sql_error insert into test.st1s2 values(now, 20); +sql_error insert into test.st1s4 using test.st1 tags(3, 'dachang') values(now, 300); sql_error select * from test.st2; sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); @@ -112,7 +114,7 @@ sql connect wxy sql reset query cache; sql select * from test.st1; -if $rows != 5 then +if $rows != 6 then return -1 endi sql_error insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20); @@ -133,7 +135,7 @@ sql connect wxy sql reset query cache; sql select * from test.st1; -if $rows != 2 then +if $rows != 6 then return -1 endi sql_error insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20); @@ -153,8 +155,9 @@ sql grant read on test.st1 with id = 1 to wxy; sql close sql connect wxy +sql reset query cache; sql select * from test.st1; -if $rows != 1 then +if $rows != 4 then return -1 endi sql_error insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20); @@ -174,8 +177,9 @@ sql grant write on test.st1 to wxy; sql close sql connect wxy +sql reset query cache; sql select * from test.st1; -if $rows != 2 then +if $rows != 6 then return -1 endi sql insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20); @@ -195,10 +199,17 @@ sql grant write on test.st1 with id = 1 to wxy; sql close sql connect wxy -sql select * from test.st1; +sql reset query cache; +sql select * from test.st1; +if $rows != 8 then + return -1 +endi sql insert into test.st1s1 values(now, 10); sql_error insert into test.st1s2 values(now, 20); -sql select * from test.st2; +sql select * from test.st2; +if $rows != 2 then + return -1 +endi sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); print =============== case 11: database write privilege and table unauthorized @@ -212,6 +223,7 @@ sql grant write on test.* to wxy; sql close sql connect wxy +sql reset query cache; sql_error select * from test.st1; sql insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20); sql_error select * from test.st2; @@ -226,7 +238,11 @@ sql grant read on test.st1 to wxy; sql close sql connect wxy +sql reset query cache; sql select * from test.st1; +if $rows != 11 then + return -1 +endi sql insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20); sql_error select * from test.st2; sql insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); @@ -241,7 +257,11 @@ sql grant read on test.st1 with id = 1 to wxy; sql close sql connect wxy +sql reset query cache; sql select * from test.st1; +if $rows != 8 then + return -1 +endi sql insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20); sql_error select * from test.st2; sql insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); @@ -256,6 +276,7 @@ sql grant write on test.st1 to wxy; sql close sql connect wxy +sql reset query cache; sql_error select * from test.st1; sql insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20); sql_error select * from test.st2; @@ -271,6 +292,7 @@ sql grant write on test.st1 with id = 1 to wxy; sql close sql connect wxy +sql reset query cache; sql_error select * from test.st1; sql insert into test.st1s1 values(now, 10); sql_error insert into test.st1s2 values(now, 20); diff --git a/tests/script/win-test-file b/tests/script/win-test-file index 2d5a1b3108..3b358993fa 100644 --- a/tests/script/win-test-file +++ b/tests/script/win-test-file @@ -3,6 +3,7 @@ ./test.sh -f tsim/user/privilege_db.sim ./test.sh -f tsim/user/privilege_sysinfo.sim ./test.sh -f tsim/user/privilege_topic.sim +./test.sh -f tsim/user/privilege_table.sim ./test.sh -f tsim/db/alter_option.sim rem ./test.sh -f tsim/db/alter_replica_13.sim ./test.sh -f tsim/db/alter_replica_31.sim From 7b7a483afc64230822519cadec80d4acfe231ee1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Apr 2023 18:43:48 +0800 Subject: [PATCH 105/109] fix(query): add tmp variable to hold the TSDBROW value. --- source/dnode/mnode/impl/src/mndMain.c | 6 +----- source/dnode/vnode/src/tsdb/tsdbRead.c | 25 +++++++++++++++---------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 5c20887cf5..92ff550895 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -124,11 +124,7 @@ static void mndCalMqRebalance(SMnode *pMnode) { int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); if (pReq != NULL) { - SRpcMsg rpcMsg = { - .msgType = TDMT_MND_TMQ_TIMER, - .pCont = pReq, - .contLen = contLen, - }; + SRpcMsg rpcMsg = { .msgType = TDMT_MND_TMQ_TIMER, .pCont = pReq, .contLen = contLen }; tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); } } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 0ad5a44919..9922312bef 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1876,10 +1876,10 @@ static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBloc TSDBROW* pRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); int64_t key = pRow->pBlockData->aTSKEY[pRow->iRow]; - int64_t ver = pRow->version; + int64_t ver = pRow->pBlockData->aVersion[pRow->iRow]; pLastBlockReader->currentKey = key; - pScanInfo->lastKeyInStt = ver; + pScanInfo->lastKeyInStt = key; if (!hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->lastBlockDelIndex, key, ver, pLastBlockReader->order, pVerRange)) { // the qualifed ts may equal to k.ts, only a greater version one. @@ -2132,16 +2132,21 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, bool mergeBlockData) { SRowMerger* pMerger = &pReader->status.merger; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); - bool copied = false; - int32_t code = TSDB_CODE_SUCCESS; - SRow* pTSRow = NULL; - TSDBROW *pRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + + int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); + bool copied = false; + int32_t code = TSDB_CODE_SUCCESS; + SRow* pTSRow = NULL; + TSDBROW* pRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + + // create local variable to hold the row value + TSDBROW fRow = {.iRow = pRow->iRow, .type = TSDBROW_COL_FMT, .pBlockData = pRow->pBlockData}; + tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", %s", pRow->pBlockData, pRow->iRow, pLastBlockReader->uid, pReader->idStr); // only last block exists if ((!mergeBlockData) || (tsLastBlock != pBlockData->aTSKEY[pDumpInfo->rowIndex])) { - code = tryCopyDistinctRowFromSttBlock(pRow, pLastBlockReader, pBlockScanInfo, tsLastBlock, pReader, &copied); + code = tryCopyDistinctRowFromSttBlock(&fRow, pLastBlockReader, pBlockScanInfo, tsLastBlock, pReader, &copied); if (code) { return code; } @@ -2150,7 +2155,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, pBlockScanInfo->lastKey = tsLastBlock; return TSDB_CODE_SUCCESS; } else { - code = tsdbRowMergerAdd(&pReader->status.merger, pRow, pReader->pSchema); + code = tsdbRowMergerAdd(&pReader->status.merger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2174,7 +2179,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, } } } else { // not merge block data - code = tsdbRowMergerAdd(&pReader->status.merger, pRow, pReader->pSchema); + code = tsdbRowMergerAdd(&pReader->status.merger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } From e9f4c769d8d753c18f79f7d3c9d077683fc57e12 Mon Sep 17 00:00:00 2001 From: xiaolei li <85657333+xleili@users.noreply.github.com> Date: Tue, 25 Apr 2023 19:20:42 +0800 Subject: [PATCH 106/109] enhance: packaging keeper in deb rpm main (#21071) * enhance: community package include taoskeeper * enhance: community rpm package add taoskeeper --- packaging/deb/DEBIAN/preinst | 37 ++++++++++++++++++++++++++++++++++ packaging/deb/DEBIAN/prerm | 1 + packaging/deb/makedeb.sh | 26 ++++++++++++++++++++++++ packaging/rpm/makerpm.sh | 24 +++++++++++++++++++++- packaging/rpm/tdengine.spec | 21 +++++++++++++++++-- packaging/tools/post.sh | 39 ++++++++++++++++++++++++++---------- packaging/tools/preun.sh | 17 +++++++++++++++- 7 files changed, 150 insertions(+), 15 deletions(-) diff --git a/packaging/deb/DEBIAN/preinst b/packaging/deb/DEBIAN/preinst index 8a1a7d4d81..904a946e20 100644 --- a/packaging/deb/DEBIAN/preinst +++ b/packaging/deb/DEBIAN/preinst @@ -26,6 +26,38 @@ if pidof taosd &> /dev/null; then sleep 1 fi +# Stop adapter service if running +if pidof taosadapter &> /dev/null; then + if pidof systemd &> /dev/null; then + ${csudo}systemctl stop taosadapter || : + elif $(which service &> /dev/null); then + ${csudo}service taosadapter stop || : + else + pid=$(ps -ef | grep "taosadapter" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi + fi + echo "Stop taosadapter service success!" + sleep 1 +fi + +# Stop keeper service if running +if pidof taoskeeper &> /dev/null; then + if pidof systemd &> /dev/null; then + ${csudo}systemctl stop taoskeeper || : + elif $(which service &> /dev/null); then + ${csudo}service taoskeeper stop || : + else + pid=$(ps -ef | grep "taoskeeper" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi + fi + echo "Stop taoskeeper service success!" + sleep 1 +fi + # if taos.cfg already softlink, remove it cfg_install_dir="/etc/taos" install_main_dir="/usr/local/taos" @@ -41,6 +73,11 @@ if [ -f "${install_main_dir}/taosadapter.service" ]; then ${csudo}rm -f ${install_main_dir}/cfg/taosadapter.service || : fi +if [ -f "${install_main_dir}/taoskeeper.toml" ]; then + ${csudo}rm -f ${install_main_dir}/cfg/taoskeeper.toml || : +fi + + # there can not libtaos.so*, otherwise ln -s error ${csudo}rm -f ${install_main_dir}/driver/libtaos.* || : [ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}rm -f ${install_main_dir}/driver/libtaosws.so || : diff --git a/packaging/deb/DEBIAN/prerm b/packaging/deb/DEBIAN/prerm index 65f261db2c..0d63115a04 100644 --- a/packaging/deb/DEBIAN/prerm +++ b/packaging/deb/DEBIAN/prerm @@ -32,6 +32,7 @@ else ${csudo}rm -f ${bin_link_dir}/udfd || : ${csudo}rm -f ${bin_link_dir}/taosadapter || : ${csudo}rm -f ${bin_link_dir}/taosdemo || : + ${csudo}rm -f ${bin_link_dir}/taoskeeper || : ${csudo}rm -f ${cfg_link_dir}/* || : ${csudo}rm -f ${inc_link_dir}/taos.h || : ${csudo}rm -f ${inc_link_dir}/taosdef.h || : diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index 94a24a4148..9f49cf345a 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -44,8 +44,31 @@ mkdir -p ${pkg_dir}${install_home_path}/include #mkdir -p ${pkg_dir}${install_home_path}/init.d mkdir -p ${pkg_dir}${install_home_path}/script +# download taoskeeper and build +if [ "$cpuType" = "x64" ] || [ "$cpuType" = "x86_64" ] || [ "$cpuType" = "amd64" ]; then + arch=amd64 +elif [ "$cpuType" = "x32" ] || [ "$cpuType" = "i386" ] || [ "$cpuType" = "i686" ]; then + arch=386 +elif [ "$cpuType" = "arm" ] || [ "$cpuType" = "aarch32" ]; then + arch=arm +elif [ "$cpuType" = "arm64" ] || [ "$cpuType" = "aarch64" ]; then + arch=arm64 +else + arch=$cpuType +fi + +echo "${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r ${arch} -e taoskeeper" +echo "$top_dir=${top_dir}" +taoskeeper_binary=`${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r $arch -e taoskeeper` +echo "taoskeeper_binary: ${taoskeeper_binary}" + +# copy config files +cp $(dirname ${taoskeeper_binary})/config/taoskeeper.toml ${pkg_dir}${install_home_path}/cfg +cp $(dirname ${taoskeeper_binary})/taoskeeper.service ${pkg_dir}${install_home_path}/cfg + cp ${compile_dir}/../packaging/cfg/taos.cfg ${pkg_dir}${install_home_path}/cfg cp ${compile_dir}/../packaging/cfg/taosd.service ${pkg_dir}${install_home_path}/cfg + if [ -f "${compile_dir}/test/cfg/taosadapter.toml" ]; then cp ${compile_dir}/test/cfg/taosadapter.toml ${pkg_dir}${install_home_path}/cfg || : fi @@ -53,6 +76,7 @@ if [ -f "${compile_dir}/test/cfg/taosadapter.service" ]; then cp ${compile_dir}/test/cfg/taosadapter.service ${pkg_dir}${install_home_path}/cfg || : fi +cp ${taoskeeper_binary} ${pkg_dir}${install_home_path}/bin #cp ${compile_dir}/../packaging/deb/taosd ${pkg_dir}${install_home_path}/init.d cp ${compile_dir}/../packaging/tools/post.sh ${pkg_dir}${install_home_path}/script cp ${compile_dir}/../packaging/tools/preun.sh ${pkg_dir}${install_home_path}/script @@ -143,6 +167,7 @@ else exit 1 fi +rm -rf ${pkg_dir}/build-taoskeeper # make deb package dpkg -b ${pkg_dir} $debname echo "make deb package success!" @@ -150,4 +175,5 @@ echo "make deb package success!" cp ${pkg_dir}/*.deb ${output_dir} # clean temp dir + rm -rf ${pkg_dir} diff --git a/packaging/rpm/makerpm.sh b/packaging/rpm/makerpm.sh index 4ac67ec754..9cf00364aa 100755 --- a/packaging/rpm/makerpm.sh +++ b/packaging/rpm/makerpm.sh @@ -35,14 +35,16 @@ function cp_rpm_package() { local cur_dir cd $1 cur_dir=$(pwd) - + echo "cp_rpm_package cd: ${cur_dir}" for dirlist in "$(ls ${cur_dir})"; do if test -d ${dirlist}; then cd ${dirlist} + echo 'cp_rpm_package ${cur_dir}/${dirlist}' cp_rpm_package ${cur_dir}/${dirlist} cd .. fi if test -e ${dirlist}; then + echo "${cur_dir}/${dirlist} ${output_dir}/TDengine-${tdengine_ver}.rpm" cp ${cur_dir}/${dirlist} ${output_dir}/TDengine-${tdengine_ver}.rpm fi done @@ -54,6 +56,25 @@ fi ${csudo}mkdir -p ${pkg_dir} cd ${pkg_dir} +# download taoskeeper and build +if [ "$cpuType" = "x64" ] || [ "$cpuType" = "x86_64" ] || [ "$cpuType" = "amd64" ]; then + arch=amd64 +elif [ "$cpuType" = "x32" ] || [ "$cpuType" = "i386" ] || [ "$cpuType" = "i686" ]; then + arch=386 +elif [ "$cpuType" = "arm" ] || [ "$cpuType" = "aarch32" ]; then + arch=arm +elif [ "$cpuType" = "arm64" ] || [ "$cpuType" = "aarch64" ]; then + arch=arm64 +else + arch=$cpuType +fi + +cd ${top_dir} +echo "${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r ${arch} -e taoskeeper" +taoskeeper_binary=`${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r $arch -e taoskeeper` +echo "taoskeeper_binary: ${taoskeeper_binary}" +cd ${package_dir} + ${csudo}mkdir -p BUILD BUILDROOT RPMS SOURCES SPECS SRPMS ${csudo}rpmbuild --define="_version ${tdengine_ver}" --define="_topdir ${pkg_dir}" --define="_compiledir ${compile_dir}" -bb ${spec_file} @@ -85,3 +106,4 @@ mv ${output_dir}/TDengine-${tdengine_ver}.rpm ${output_dir}/${rpmname} cd .. ${csudo}rm -rf ${pkg_dir} +rm -rf ${top_dir}/build-taoskeeper \ No newline at end of file diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index c21063e6a4..52d5335003 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -3,6 +3,7 @@ %define cfg_install_dir /etc/taos %define __strip /bin/true %global __python /usr/bin/python3 +%global _build_id_links none Name: tdengine Version: %{_version} @@ -62,6 +63,15 @@ fi if [ -f %{_compiledir}/test/cfg/taosadapter.service ]; then cp %{_compiledir}/test/cfg/taosadapter.service %{buildroot}%{homepath}/cfg fi + +if [ -f %{_compiledir}/../build-taoskeeper/config/taoskeeper.toml ]; then + cp %{_compiledir}/../build-taoskeeper/config/taoskeeper.toml %{buildroot}%{homepath}/cfg ||: +fi + +if [ -f %{_compiledir}/../build-taoskeeper/taoskeeper.service ]; then + cp %{_compiledir}/../build-taoskeeper/taoskeeper.service %{buildroot}%{homepath}/cfg ||: +fi + #cp %{_compiledir}/../packaging/rpm/taosd %{buildroot}%{homepath}/init.d cp %{_compiledir}/../packaging/tools/post.sh %{buildroot}%{homepath}/script cp %{_compiledir}/../packaging/tools/preun.sh %{buildroot}%{homepath}/script @@ -73,8 +83,12 @@ cp %{_compiledir}/build/bin/taosd %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/udfd %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taosBenchmark %{buildroot}%{homepath}/bin +if [ -f %{_compiledir}/../build-taoskeeper/taoskeeper ]; then + cp %{_compiledir}/../build-taoskeeper/taoskeeper %{buildroot}%{homepath}/bin +fi + if [ -f %{_compiledir}/build/bin/taosadapter ]; then - cp %{_compiledir}/build/bin/taosadapter %{buildroot}%{homepath}/bin ||: + cp %{_compiledir}/build/bin/taosadapter %{buildroot}%{homepath}/bin fi cp %{_compiledir}/build/lib/${libfile} %{buildroot}%{homepath}/driver [ -f %{_compiledir}/build/lib/${wslibfile} ] && cp %{_compiledir}/build/lib/${wslibfile} %{buildroot}%{homepath}/driver ||: @@ -119,7 +133,9 @@ if [ -f %{_compiledir}/build/bin/jemalloc-config ]; then cp %{_compiledir}/build/lib/pkgconfig/jemalloc.pc %{buildroot}%{homepath}/jemalloc/lib/pkgconfig fi fi - +ls -al %{buildroot}%{homepath}/bin +tree -L 5 +echo "==============================copying files done" #Scripts executed before installation %pre if [ -f /var/lib/taos/dnode/dnodeCfg.json ]; then @@ -196,6 +212,7 @@ if [ $1 -eq 0 ];then ${csudo}rm -f ${bin_link_dir}/taosd || : ${csudo}rm -f ${bin_link_dir}/udfd || : ${csudo}rm -f ${bin_link_dir}/taosadapter || : + ${csudo}rm -f ${bin_link_dir}/taoskeeper || : ${csudo}rm -f ${cfg_link_dir}/* || : ${csudo}rm -f ${inc_link_dir}/taos.h || : ${csudo}rm -f ${inc_link_dir}/taosdef.h || : diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index 3a013ade2c..fc392c9684 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -436,7 +436,7 @@ function local_fqdn_check() { function install_taosadapter_config() { if [ ! -f "${cfg_install_dir}/taosadapter.toml" ]; then - [ ! -d %{cfg_install_dir} ] && + [ ! -d ${cfg_install_dir} ] && ${csudo}${csudo}mkdir -p ${cfg_install_dir} [ -f ${cfg_dir}/taosadapter.toml ] && ${csudo}cp ${cfg_dir}/taosadapter.toml ${cfg_install_dir} [ -f ${cfg_install_dir}/taosadapter.toml ] && @@ -451,19 +451,26 @@ function install_taosadapter_config() { } function install_taoskeeper_config() { - if [ ! -f "${cfg_install_dir}/keeper.toml" ]; then - [ ! -d %{cfg_install_dir} ] && - ${csudo}${csudo}mkdir -p ${cfg_install_dir} - [ -f ${cfg_dir}/keeper.toml ] && ${csudo}cp ${cfg_dir}/keeper.toml ${cfg_install_dir} - [ -f ${cfg_install_dir}/keeper.toml ] && - ${csudo}chmod 644 ${cfg_install_dir}/keeper.toml + # if new environment without taoskeeper + if [[ ! -f "${cfg_install_dir}/keeper.toml" ]] && [[ ! -f "${cfg_install_dir}/taoskeeper.toml" ]]; then + [ ! -d ${cfg_install_dir} ] && ${csudo}${csudo}mkdir -p ${cfg_install_dir} + [ -f ${cfg_dir}/taoskeeper.toml ] && ${csudo}cp ${cfg_dir}/taoskeeper.toml ${cfg_install_dir} + [ -f ${cfg_install_dir}/taoskeeper.toml ] && + ${csudo}chmod 644 ${cfg_install_dir}/taoskeeper.toml + fi + # if old machine with taoskeeper.toml file + if [ -f ${cfg_install_dir}/taoskeeper.toml ]; then + ${csudo}mv ${cfg_dir}/taoskeeper.toml ${cfg_dir}/taoskeeper.toml.new fi - [ -f ${cfg_dir}/keeper.toml ] && - ${csudo}mv ${cfg_dir}/keeper.toml ${cfg_dir}/keeper.toml.new + if [ -f ${cfg_install_dir}/keeper.toml ]; then + echo "The file keeper.toml will be renamed to taoskeeper.toml" + ${csudo}mv ${cfg_install_dir}/keeper.toml ${cfg_install_dir}/taoskeeper.toml + ${csudo}mv ${cfg_dir}/taoskeeper.toml ${cfg_dir}/taoskeeper.toml.new + fi - [ -f ${cfg_install_dir}/keeper.toml ] && - ${csudo}ln -s ${cfg_install_dir}/keeper.toml ${cfg_dir} + [ -f ${cfg_install_dir}/taoskeeper.toml ] && + ${csudo}ln -s ${cfg_install_dir}/taoskeeper.toml ${cfg_dir} } function install_config() { @@ -655,6 +662,15 @@ function install_taosadapter_service() { fi } +function install_taoskeeper_service() { + if ((${service_mod}==0)); then + [ -f ${script_dir}/../cfg/taoskeeper.service ] &&\ + ${csudo}cp ${script_dir}/../cfg/taoskeeper.service \ + ${service_config_dir}/ || : + ${csudo}systemctl daemon-reload + fi +} + function install_service() { log_print "start install service" if [ "$osType" != "Darwin" ]; then @@ -732,6 +748,7 @@ function install_TDengine() { install_taosadapter_config install_taoskeeper_config install_taosadapter_service + install_taoskeeper_service install_service install_app diff --git a/packaging/tools/preun.sh b/packaging/tools/preun.sh index 0e96c71d5d..68f6b53c45 100755 --- a/packaging/tools/preun.sh +++ b/packaging/tools/preun.sh @@ -17,7 +17,7 @@ cfg_link_dir="/usr/local/taos/cfg" service_config_dir="/etc/systemd/system" taos_service_name="taosd" - +taoskeeper_service_name="taoskeeper" csudo="" if command -v sudo > /dev/null; then csudo="sudo " @@ -57,6 +57,13 @@ function kill_taosd() { fi } +function kill_taoskeeper() { + pid=$(ps -ef | grep "taoskeeper" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi +} + function clean_service_on_systemd() { taosadapter_service_config="${service_config_dir}/taosadapter.service" if systemctl is-active --quiet taosadapter; then @@ -76,6 +83,12 @@ function clean_service_on_systemd() { [ -f ${taosadapter_service_config} ] && ${csudo}rm -f ${taosadapter_service_config} + taoskeeper_service_config="${service_config_dir}/${taoskeeper_service_name}.service" + if systemctl is-active --quiet ${taoskeeper_service_name}; then + echo "TDengine taoskeeper is running, stopping it..." + ${csudo}systemctl stop ${taoskeeper_service_name} &> /dev/null || echo &> /dev/null + fi + [ -f ${taoskeeper_service_config} ] && ${csudo}rm -f ${taoskeeper_service_config} } function clean_service_on_sysvinit() { @@ -111,6 +124,7 @@ function clean_service() { # must manual stop taosd kill_taosadapter kill_taosd + kill_taoskeeper fi } @@ -124,6 +138,7 @@ ${csudo}rm -f ${bin_link_dir}/taosadapter || : ${csudo}rm -f ${bin_link_dir}/taosBenchmark || : ${csudo}rm -f ${bin_link_dir}/taosdemo || : ${csudo}rm -f ${bin_link_dir}/set_core || : +${csudo}rm -f ${bin_link_dir}/taoskeeper || : ${csudo}rm -f ${cfg_link_dir}/*.new || : ${csudo}rm -f ${inc_link_dir}/taos.h || : ${csudo}rm -f ${inc_link_dir}/taosdef.h || : From 2a45ef88b00db98d307bf7357c15ef559aef7b50 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 26 Apr 2023 00:40:33 +0800 Subject: [PATCH 107/109] fix(query): fix memory leak. --- source/client/src/clientTmq.c | 6 +----- source/dnode/vnode/src/tsdb/tsdbRead.c | 14 +++++++------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 16a4f55840..9292be83e9 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1664,11 +1664,7 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p return handleErrorBeforePoll(pVg, pTmq); } - sendInfo->msgInfo = (SDataBuf){ - .pData = msg, - .len = msgSize, - .handle = NULL, - }; + sendInfo->msgInfo = (SDataBuf){ .pData = msg, .len = msgSize, .handle = NULL }; sendInfo->requestId = req.reqId; sendInfo->requestObjRefId = 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 9922312bef..ad28b5c9e6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -789,12 +789,6 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd goto _end; } - pReader->status.pLDataIter = taosMemoryCalloc(pVnode->config.sttTrigger, sizeof(SLDataIter)); - if (pReader->status.pLDataIter == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _end; - } - ASSERT (pReader->suppInfo.colId[0] == PRIMARYKEY_TIMESTAMP_COL_ID); pReader->status.pPrimaryTsCol = taosArrayGet(pResBlock->pDataBlock, pSup->slotId[0]); @@ -4388,6 +4382,12 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL goto _err; } + pReader->status.pLDataIter = taosMemoryCalloc(pVnode->config.sttTrigger, sizeof(SLDataIter)); + if (pReader->status.pLDataIter == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + pReader->suspended = true; if (countOnly) { @@ -4487,7 +4487,7 @@ void tsdbReaderClose(STsdbReader* pReader) { tsdbUninitReaderLock(pReader); taosMemoryFreeClear(pReader->status.pLDataIter); - taosMemoryFree(pReader->status.uidList.tableUidList); + taosMemoryFreeClear(pReader->status.uidList.tableUidList); SIOCostSummary* pCost = &pReader->cost; SFilesetIter* pFilesetIter = &pReader->status.fileIter; From 8e2edd6ebf1b6613b6d95c0db5cc9373a39328fd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 26 Apr 2023 10:00:00 +0800 Subject: [PATCH 108/109] fix(query): fix null ptr access. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index ad28b5c9e6..e643e8f1a4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -790,7 +790,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd } ASSERT (pReader->suppInfo.colId[0] == PRIMARYKEY_TIMESTAMP_COL_ID); - pReader->status.pPrimaryTsCol = taosArrayGet(pResBlock->pDataBlock, pSup->slotId[0]); + pReader->status.pPrimaryTsCol = taosArrayGet(pReader->pResBlock->pDataBlock, pSup->slotId[0]); tsdbInitReaderLock(pReader); From 5db5205e31733ca2a991144575f4bcb885279054 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 26 Apr 2023 13:41:16 +0800 Subject: [PATCH 109/109] fix: schema/tag version update to int32 --- include/libs/catalog/catalog.h | 4 ++-- include/libs/qcom/query.h | 4 ++-- source/libs/executor/src/tsort.c | 4 ++-- source/libs/qcom/src/querymsg.c | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 429e7ffa73..d7084cfac4 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -124,8 +124,8 @@ typedef struct SSTableVersion { char stbName[TSDB_TABLE_NAME_LEN]; uint64_t dbId; uint64_t suid; - int16_t sversion; - int16_t tversion; + int32_t sversion; + int32_t tversion; int32_t smaVer; } SSTableVersion; diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index cfc6ef2025..3841210076 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -116,8 +116,8 @@ typedef struct STableMeta { // if the table is TSDB_CHILD_TABLE, the following information is acquired from the corresponding super table meta // info - int16_t sversion; - int16_t tversion; + int32_t sversion; + int32_t tversion; STableComInfo tableInfo; SSchema schema[]; } STableMeta; diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 6c8e581b3f..d973b5bbf3 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -155,7 +155,7 @@ void tsortDestroySortHandle(SSortHandle* pSortHandle) { int64_t fetchUs = 0, fetchNum = 0; tsortClearOrderdSource(pSortHandle->pOrderedSource, &fetchUs, &fetchNum); - qError("all source fetch time: %" PRId64 "us num:%" PRId64 " %s", fetchUs, fetchNum, pSortHandle->idStr); + qDebug("all source fetch time: %" PRId64 "us num:%" PRId64 " %s", fetchUs, fetchNum, pSortHandle->idStr); taosArrayDestroy(pSortHandle->pOrderedSource); taosMemoryFreeClear(pSortHandle); @@ -316,7 +316,7 @@ static int32_t sortComparInit(SMsortComparParam* pParam, SArray* pSources, int32 } int64_t et = taosGetTimestampUs(); - qError("init for merge sort completed, elapsed time:%.2f ms, %s", (et - st) / 1000.0, pHandle->idStr); + qDebug("init for merge sort completed, elapsed time:%.2f ms, %s", (et - st) / 1000.0, pHandle->idStr); } return code; diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index b62a3e4932..01b136d5e0 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -407,7 +407,7 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp *msg, bool isStb, STableMeta * pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes; } - qDebug("table %s uid %" PRIx64 " meta returned, type %d vgId:%d db %s stb %s suid %" PRIx64 " sver %d tver %d" PRIx64 + qDebug("table %s uid %" PRIx64 " meta returned, type %d vgId:%d db %s stb %s suid %" PRIx64 " sver %d tver %d" " tagNum %d colNum %d precision %d rowSize %d", msg->tbName, pTableMeta->uid, pTableMeta->tableType, pTableMeta->vgId, msg->dbFName, msg->stbName, pTableMeta->suid, pTableMeta->sversion, pTableMeta->tversion, pTableMeta->tableInfo.numOfTags,