diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 6a3439ada9..b75d4607b8 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 125c77a + GIT_TAG 285b5e0 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/docs/en/12-taos-sql/06-select.md b/docs/en/12-taos-sql/06-select.md index 118a248373..e409973173 100644 --- a/docs/en/12-taos-sql/06-select.md +++ b/docs/en/12-taos-sql/06-select.md @@ -66,7 +66,7 @@ order_expr: A query can be performed on some or all columns. Data and tag columns can all be included in the SELECT list. -## Wildcards +### Wildcards You can use an asterisk (\*) as a wildcard character to indicate all columns. For standard tables, the asterisk indicates only data columns. For supertables and subtables, tag columns are also included. @@ -136,6 +136,8 @@ taos> SELECT ts, ts AS primary_key_ts FROM d1001; ### Pseudocolumns +**Pseudocolumn:** A pseudo-column behaves like a table column but is not actually stored in the table. You can select from pseudo-columns, but you cannot insert, update, or delete their values. A pseudo-column is also similar to a function without arguments. This section describes these pseudo-columns: + **TBNAME** The TBNAME pseudocolumn in a supertable contains the names of subtables within the supertable. diff --git a/docs/zh/12-taos-sql/06-select.md b/docs/zh/12-taos-sql/06-select.md index c49351b0c9..5841904e61 100644 --- a/docs/zh/12-taos-sql/06-select.md +++ b/docs/zh/12-taos-sql/06-select.md @@ -137,6 +137,8 @@ taos> SELECT ts, ts AS primary_key_ts FROM d1001; ### 伪列 +**伪列**: 伪列的行为表现与普通数据列相似但其并不实际存储在表中。可以查询伪列,但不能对其做插入、更新和删除的操作。伪列有点像没有参数的函数。下面介绍是可用的伪列: + **TBNAME** `TBNAME` 可以视为超级表中一个特殊的标签,代表子表的表名。 diff --git a/include/client/taos.h b/include/client/taos.h index 49cfbb52b8..270b647a77 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -254,7 +254,7 @@ enum tmq_res_t { TMQ_RES_INVALID = -1, TMQ_RES_DATA = 1, TMQ_RES_TABLE_META = 2, - TMQ_RES_TAOSX = 3, + TMQ_RES_METADATA = 3, }; typedef struct tmq_raw_data { diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 2de4ffdc17..66bae5ad3b 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -120,6 +120,7 @@ extern SDiskCfg tsDiskCfg[]; // udf extern bool tsStartUdfd; +extern char tsUdfdResFuncs[]; // schemaless extern char tsSmlChildTableName[]; diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index 80fa7a7218..849d83a58b 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -34,7 +34,7 @@ typedef struct { TXN txn; } SStreamState; -SStreamState* streamStateOpen(char* path, SStreamTask* pTask); +SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath); void streamStateClose(SStreamState* pState); int32_t streamStateBegin(SStreamState* pState); int32_t streamStateCommit(SStreamState* pState); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 01d42d6950..840e7309fe 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -619,6 +619,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_RSMA_EMPTY_INFO TAOS_DEF_ERROR_CODE(0, 0x3156) #define TSDB_CODE_RSMA_INVALID_SCHEMA TAOS_DEF_ERROR_CODE(0, 0x3157) #define TSDB_CODE_RSMA_REGEX_MATCH TAOS_DEF_ERROR_CODE(0, 0x3158) +#define TSDB_CODE_RSMA_STREAM_STATE_OPEN TAOS_DEF_ERROR_CODE(0, 0x3159) +#define TSDB_CODE_RSMA_STREAM_STATE_COMMIT TAOS_DEF_ERROR_CODE(0, 0x3160) //index #define TSDB_CODE_INDEX_REBUILDING TAOS_DEF_ERROR_CODE(0, 0x3200) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index b8fa9580e7..574d8188fe 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -52,7 +52,7 @@ enum { RES_TYPE__QUERY = 1, RES_TYPE__TMQ, RES_TYPE__TMQ_META, - RES_TYPE__TAOSX, + RES_TYPE__TMQ_METADATA, }; #define SHOW_VARIABLES_RESULT_COLS 2 @@ -60,9 +60,9 @@ enum { #define SHOW_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE) #define TD_RES_QUERY(res) (*(int8_t*)res == RES_TYPE__QUERY) -#define TD_RES_TMQ(res) (*(int8_t*)res == RES_TYPE__TMQ || *(int8_t*)res == RES_TYPE__TAOSX) +#define TD_RES_TMQ(res) (*(int8_t*)res == RES_TYPE__TMQ) #define TD_RES_TMQ_META(res) (*(int8_t*)res == RES_TYPE__TMQ_META) -#define TD_RES_TMQ_TAOSX(res) (*(int8_t*)res == RES_TYPE__TAOSX) +#define TD_RES_TMQ_METADATA(res) (*(int8_t*)res == RES_TYPE__TMQ_METADATA) typedef struct SAppInstInfo SAppInstInfo; diff --git a/source/client/src/TMQConnector.c b/source/client/src/TMQConnector.c index fcf6957df9..26bf55055f 100644 --- a/source/client/src/TMQConnector.c +++ b/source/client/src/TMQConnector.c @@ -212,7 +212,7 @@ JNIEXPORT void JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqCommitAsync(JN tmq_commit_async(tmq, res, commit_cb, consumer); } -JNIEXPORT int JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqUnsubscribeImp(JNIEnv *env, jobject jobj, jlong jtmq) { +JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqUnsubscribeImp(JNIEnv *env, jobject jobj, jlong jtmq) { tmq_t *tmq = (tmq_t *)jtmq; if (tmq == NULL) { jniError("jobj:%p, tmq is closed", jobj); @@ -222,7 +222,7 @@ JNIEXPORT int JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqUnsubscribeImp( return tmq_unsubscribe((tmq_t *)tmq); } -JNIEXPORT int JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqConsumerCloseImp(JNIEnv *env, jobject jobj, +JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqConsumerCloseImp(JNIEnv *env, jobject jobj, jlong jtmq) { tmq_t *tmq = (tmq_t *)jtmq; if (tmq == NULL) { diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 7ce80553a0..75ccd44977 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -878,12 +878,18 @@ int hbMgrInit() { clientHbMgr.appHbMgrs = taosArrayInit(0, sizeof(void *)); TdThreadMutexAttr attr = {0}; - taosThreadMutexAttrSetType(&attr, PTHREAD_MUTEX_RECURSIVE); + int ret = taosThreadMutexAttrInit(&attr); assert(ret == 0); - taosThreadMutexInit(&clientHbMgr.lock, &attr); - taosThreadMutexAttrDestroy(&attr); + ret = taosThreadMutexAttrSetType(&attr, PTHREAD_MUTEX_RECURSIVE); + assert(ret == 0); + + ret = taosThreadMutexInit(&clientHbMgr.lock, &attr); + assert(ret == 0); + + ret = taosThreadMutexAttrDestroy(&attr); + assert(ret == 0); // init handle funcs hbMgrInitHandle(); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 0e1d82b273..5128695359 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -189,6 +189,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, tscError("%d failed to add to request container, reqId:0x%" PRIx64 ", conn:%d, %s", (*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql); + taosMemoryFree(param); destroyRequest(*pRequest); *pRequest = NULL; return TSDB_CODE_TSC_OUT_OF_MEMORY; diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 73636e7372..13cbaa0e22 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -148,7 +148,7 @@ int taos_errno(TAOS_RES *res) { return terrno; } - if (TD_RES_TMQ(res)) { + if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { return 0; } @@ -162,7 +162,7 @@ const char *taos_errstr(TAOS_RES *res) { return (const char *)tstrerror(terrno); } - if (TD_RES_TMQ(res)) { + if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { return "success"; } @@ -184,7 +184,7 @@ void taos_free_result(TAOS_RES *res) { SRequestObj *pRequest = (SRequestObj *)res; tscDebug("0x%" PRIx64 " taos_free_result start to free query", pRequest->requestId); destroyRequest(pRequest); - } else if (TD_RES_TMQ_TAOSX(res)) { + } else if (TD_RES_TMQ_METADATA(res)) { SMqTaosxRspObj *pRsp = (SMqTaosxRspObj *)res; if (pRsp->rsp.blockData) taosArrayDestroyP(pRsp->rsp.blockData, taosMemoryFree); if (pRsp->rsp.blockDataLen) taosArrayDestroy(pRsp->rsp.blockDataLen); @@ -264,7 +264,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { return doFetchRows(pRequest, true, true); #endif - } else if (TD_RES_TMQ(res)) { + } else if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { SMqRspObj *msg = ((SMqRspObj *)res); SReqResultInfo *pResultInfo; if (msg->resIter == -1) { @@ -437,7 +437,7 @@ const char *taos_data_type(int type) { const char *taos_get_client_info() { return version; } int taos_affected_rows(TAOS_RES *res) { - if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res)) { + if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res)) { return 0; } @@ -454,7 +454,7 @@ int taos_result_precision(TAOS_RES *res) { if (TD_RES_QUERY(res)) { SRequestObj *pRequest = (SRequestObj *)res; return pRequest->body.resInfo.precision; - } else if (TD_RES_TMQ(res)) { + } else if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { SReqResultInfo *info = tmqGetCurResInfo(res); return info->precision; } @@ -487,7 +487,7 @@ int taos_select_db(TAOS *taos, const char *db) { } void taos_stop_query(TAOS_RES *res) { - if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res)) { + if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res)) { return; } @@ -559,7 +559,7 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) { (*rows) = pResultInfo->row; (*numOfRows) = pResultInfo->numOfRows; return pRequest->code; - } else if (TD_RES_TMQ(res)) { + } else if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { SReqResultInfo *pResultInfo = tmqGetNextResInfo(res, true); if (pResultInfo == NULL) return -1; @@ -578,7 +578,7 @@ int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) { return 0; } - if (TD_RES_TMQ(res)) { + if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { SReqResultInfo *pResultInfo = tmqGetNextResInfo(res, false); if (pResultInfo == NULL) { (*numOfRows) = 0; diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index c135965f07..eb7b45cc05 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -30,7 +30,7 @@ static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* sch char* string = NULL; cJSON* json = cJSON_CreateObject(); if (json == NULL) { - return string; + return NULL; } cJSON* type = cJSON_CreateString("create"); cJSON_AddItemToObject(json, "type", type); @@ -39,10 +39,10 @@ static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* sch // sprintf(uid, "%"PRIi64, id); // cJSON* id_ = cJSON_CreateString(uid); // cJSON_AddItemToObject(json, "id", id_); - cJSON* tableName = cJSON_CreateString(name); - cJSON_AddItemToObject(json, "tableName", tableName); cJSON* tableType = cJSON_CreateString(t == TSDB_NORMAL_TABLE ? "normal" : "super"); cJSON_AddItemToObject(json, "tableType", tableType); + cJSON* tableName = cJSON_CreateString(name); + cJSON_AddItemToObject(json, "tableName", tableName); // cJSON* version = cJSON_CreateNumber(1); // cJSON_AddItemToObject(json, "version", version); @@ -112,10 +112,10 @@ static char* buildAlterSTableJson(void* alterData, int32_t alterDataLen) { // cJSON_AddItemToObject(json, "uid", uid); SName name = {0}; tNameFromString(&name, req.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - cJSON* tableName = cJSON_CreateString(name.tname); - cJSON_AddItemToObject(json, "tableName", tableName); cJSON* tableType = cJSON_CreateString("super"); cJSON_AddItemToObject(json, "tableType", tableType); + cJSON* tableName = cJSON_CreateString(name.tname); + cJSON_AddItemToObject(json, "tableName", tableName); cJSON* alterType = cJSON_CreateNumber(req.alterType); cJSON_AddItemToObject(json, "alterType", alterType); @@ -199,8 +199,6 @@ static char* processCreateStb(SMqMetaRsp* metaRsp) { goto _err; } string = buildCreateTableJson(&req.schemaRow, &req.schemaTag, req.name, req.suid, TSDB_SUPER_TABLE); - tDecoderClear(&coder); - return string; _err: tDecoderClear(&coder); @@ -221,32 +219,22 @@ static char* processAlterStb(SMqMetaRsp* metaRsp) { goto _err; } string = buildAlterSTableJson(req.alterOriData, req.alterOriDataLen); - tDecoderClear(&coder); - return string; _err: tDecoderClear(&coder); return string; } -static char* buildCreateCTableJson(STag* pTag, char* sname, char* name, SArray* tagName, int64_t id, uint8_t tagNum) { - char* string = NULL; - SArray* pTagVals = NULL; - cJSON* json = cJSON_CreateObject(); - if (json == NULL) { - return string; - } - cJSON* type = cJSON_CreateString("create"); - cJSON_AddItemToObject(json, "type", type); - // char cid[32] = {0}; - // sprintf(cid, "%"PRIi64, id); - // cJSON* cid_ = cJSON_CreateString(cid); - // cJSON_AddItemToObject(json, "id", cid_); +static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq){ + STag* pTag = (STag*)pCreateReq->ctb.pTag; + char* sname = pCreateReq->ctb.name; + char* name = pCreateReq->name; + SArray* tagName = pCreateReq->ctb.tagName; + int64_t id = pCreateReq->uid; + uint8_t tagNum = pCreateReq->ctb.tagNum; cJSON* tableName = cJSON_CreateString(name); cJSON_AddItemToObject(json, "tableName", tableName); - cJSON* tableType = cJSON_CreateString("child"); - cJSON_AddItemToObject(json, "tableType", tableType); cJSON* using = cJSON_CreateString(sname); cJSON_AddItemToObject(json, "using", using); cJSON* tagNumJson = cJSON_CreateNumber(tagNum); @@ -255,6 +243,7 @@ static char* buildCreateCTableJson(STag* pTag, char* sname, char* name, SArray* // cJSON_AddItemToObject(json, "version", version); cJSON* tags = cJSON_CreateArray(); + SArray* pTagVals = NULL; int32_t code = tTagToValArray(pTag, &pTagVals); if (code) { goto end; @@ -313,11 +302,37 @@ static char* buildCreateCTableJson(STag* pTag, char* sname, char* name, SArray* cJSON_AddItemToArray(tags, tag); } -end: + end: cJSON_AddItemToObject(json, "tags", tags); + taosArrayDestroy(pTagVals); +} + +static char* buildCreateCTableJson(SVCreateTbReq* pCreateReq, int32_t nReqs) { + char* string = NULL; + cJSON* json = cJSON_CreateObject(); + if (json == NULL) { + return NULL; + } + cJSON* type = cJSON_CreateString("create"); + cJSON_AddItemToObject(json, "type", type); + // char cid[32] = {0}; + // sprintf(cid, "%"PRIi64, id); + // cJSON* cid_ = cJSON_CreateString(cid); + // cJSON_AddItemToObject(json, "id", cid_); + + cJSON* tableType = cJSON_CreateString("child"); + cJSON_AddItemToObject(json, "tableType", tableType); + + buildChildElement(json, pCreateReq); + cJSON* createList = cJSON_CreateArray(); + for(int i = 0; nReqs > 1 && i < nReqs; i++){ + cJSON* create = cJSON_CreateObject(); + buildChildElement(create, pCreateReq + i); + cJSON_AddItemToArray(createList, create); + } + cJSON_AddItemToObject(json, "createList", createList); string = cJSON_PrintUnformatted(json); cJSON_Delete(json); - taosArrayDestroy(pTagVals); return string; } @@ -335,21 +350,58 @@ static char* processCreateTable(SMqMetaRsp* metaRsp) { } // loop to create table - for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { - pCreateReq = req.pReqs + iReq; + if (req.nReqs > 0) { + pCreateReq = req.pReqs; if (pCreateReq->type == TSDB_CHILD_TABLE) { - string = buildCreateCTableJson((STag*)pCreateReq->ctb.pTag, pCreateReq->ctb.name, pCreateReq->name, - pCreateReq->ctb.tagName, pCreateReq->uid, pCreateReq->ctb.tagNum); + string = buildCreateCTableJson(req.pReqs, req.nReqs); } else if (pCreateReq->type == TSDB_NORMAL_TABLE) { - string = - buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid, TSDB_NORMAL_TABLE); + string = buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid, TSDB_NORMAL_TABLE); } } +_exit: + for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { + pCreateReq = req.pReqs + iReq; + taosMemoryFreeClear(pCreateReq->comment); + if (pCreateReq->type == TSDB_CHILD_TABLE) { + taosArrayDestroy(pCreateReq->ctb.tagName); + } + } tDecoderClear(&decoder); + return string; +} + +static char* processAutoCreateTable(STaosxRsp* rsp) { + ASSERT(rsp->createTableNum != 0); + + SDecoder* decoder = taosMemoryCalloc(rsp->createTableNum, sizeof(SDecoder)); + SVCreateTbReq* pCreateReq = taosMemoryCalloc(rsp->createTableNum, sizeof(SVCreateTbReq)); + char* string = NULL; + + // loop to create table + for (int32_t iReq = 0; iReq < rsp->createTableNum; iReq++) { + // decode + void** data = taosArrayGet(rsp->createTableReq, iReq); + int32_t *len = taosArrayGet(rsp->createTableLen, iReq); + tDecoderInit(&decoder[iReq], *data, *len); + if (tDecodeSVCreateTbReq(&decoder[iReq], pCreateReq + iReq) < 0) { + goto _exit; + } + + ASSERT(pCreateReq[iReq].type == TSDB_CHILD_TABLE); + } + string = buildCreateCTableJson(pCreateReq, rsp->createTableNum); _exit: - tDecoderClear(&decoder); + for(int i = 0; i < rsp->createTableNum; i++){ + tDecoderClear(&decoder[i]); + taosMemoryFreeClear(pCreateReq[i].comment); + if (pCreateReq[i].type == TSDB_CHILD_TABLE) { + taosArrayDestroy(pCreateReq[i].ctb.tagName); + } + } + taosMemoryFree(decoder); + taosMemoryFree(pCreateReq); return string; } @@ -374,10 +426,10 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) { cJSON_AddItemToObject(json, "type", type); // cJSON* uid = cJSON_CreateNumber(id); // cJSON_AddItemToObject(json, "uid", uid); - cJSON* tableName = cJSON_CreateString(vAlterTbReq.tbName); - cJSON_AddItemToObject(json, "tableName", tableName); cJSON* tableType = cJSON_CreateString(vAlterTbReq.action == TSDB_ALTER_TABLE_UPDATE_TAG_VAL ? "child" : "normal"); cJSON_AddItemToObject(json, "tableType", tableType); + cJSON* tableName = cJSON_CreateString(vAlterTbReq.tbName); + cJSON_AddItemToObject(json, "tableName", tableName); cJSON* alterType = cJSON_CreateNumber(vAlterTbReq.action); cJSON_AddItemToObject(json, "alterType", alterType); @@ -462,6 +514,7 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) { string = cJSON_PrintUnformatted(json); _exit: + cJSON_Delete(json); tDecoderClear(&decoder); return string; } @@ -485,14 +538,15 @@ static char* processDropSTable(SMqMetaRsp* metaRsp) { } cJSON* type = cJSON_CreateString("drop"); cJSON_AddItemToObject(json, "type", type); - cJSON* tableName = cJSON_CreateString(req.name); - cJSON_AddItemToObject(json, "tableName", tableName); cJSON* tableType = cJSON_CreateString("super"); cJSON_AddItemToObject(json, "tableType", tableType); + cJSON* tableName = cJSON_CreateString(req.name); + cJSON_AddItemToObject(json, "tableName", tableName); string = cJSON_PrintUnformatted(json); _exit: + cJSON_Delete(json); tDecoderClear(&decoder); return string; } @@ -533,6 +587,7 @@ static char* processDropTable(SMqMetaRsp* metaRsp) { string = cJSON_PrintUnformatted(json); _exit: + cJSON_Delete(json); tDecoderClear(&decoder); return string; } @@ -549,6 +604,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { goto end; } + pRequest->syncQuery = true; if (!pRequest->pDb) { code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; goto end; @@ -637,6 +693,7 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { goto end; } + pRequest->syncQuery = true; if (!pRequest->pDb) { code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; goto end; @@ -717,6 +774,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { goto end; } + pRequest->syncQuery = true; if (!pRequest->pDb) { code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; goto end; @@ -830,6 +888,14 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { code = pRequest->code; end: + for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { + pCreateReq = req.pReqs + iReq; + taosMemoryFreeClear(pCreateReq->comment); + if (pCreateReq->type == TSDB_CHILD_TABLE) { + taosArrayDestroy(pCreateReq->ctb.tagName); + } + } + taosHashCleanup(pVgroupHashmap); destroyRequest(pRequest); tDecoderClear(&coder); @@ -860,7 +926,7 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { if (code != TSDB_CODE_SUCCESS) { goto end; } - + pRequest->syncQuery = true; if (!pRequest->pDb) { code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; goto end; @@ -1033,6 +1099,7 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { goto end; } + pRequest->syncQuery = true; if (!pRequest->pDb) { code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; goto end; @@ -1152,6 +1219,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) goto end; } + pRequest->syncQuery = true; if (!pRequest->pDb) { uError("WriteRaw:not use db"); code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; @@ -1339,6 +1407,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { return terrno; } + pRequest->syncQuery = true; rspObj.resIter = -1; rspObj.resType = RES_TYPE__TMQ; @@ -1529,6 +1598,8 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { subReq->length += sizeof(SSubmitBlk) + schemaLen + totalLen; subReq->numOfBlocks++; taosMemoryFreeClear(pTableMeta); + rspObj.resInfo.pRspMsg = NULL; + doFreeReqResultInfo(&rspObj.resInfo); } pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); @@ -1578,6 +1649,313 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { code = pRequest->code; end: + tDeleteSMqDataRsp(&rspObj.rsp); + rspObj.resInfo.pRspMsg = NULL; + doFreeReqResultInfo(&rspObj.resInfo); + tDecoderClear(&decoder); + qDestroyQuery(pQuery); + destroyRequest(pRequest); + taosHashCleanup(pVgHash); + taosMemoryFreeClear(pTableMeta); + return code; +} + +static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) { + int32_t code = TSDB_CODE_SUCCESS; + SHashObj* pVgHash = NULL; + SQuery* pQuery = NULL; + SMqTaosxRspObj rspObj = {0}; + SDecoder decoder = {0}; + STableMeta* pTableMeta = NULL; + + terrno = TSDB_CODE_SUCCESS; + SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT); + if (!pRequest) { + uError("WriteRaw:createRequest error request is null"); + return terrno; + } + + pRequest->syncQuery = true; + rspObj.resIter = -1; + rspObj.resType = RES_TYPE__TMQ_METADATA; + + tDecoderInit(&decoder, data, dataLen); + code = tDecodeSTaosxRsp(&decoder, &rspObj.rsp); + if (code != 0) { + uError("WriteRaw:decode smqDataRsp error"); + code = TSDB_CODE_INVALID_MSG; + goto end; + } + + if (!pRequest->pDb) { + uError("WriteRaw:not use db"); + code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; + goto end; + } + + pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + taosHashSetFreeFp(pVgHash, destroyVgHash); + struct SCatalog* pCatalog = NULL; + code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw: get gatlog error"); + goto end; + } + + SRequestConnInfo conn = {0}; + conn.pTrans = pRequest->pTscObj->pAppInfo->pTransporter; + conn.requestId = pRequest->requestId; + conn.requestObjRefId = pRequest->self; + conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); + + printf("raw data block num:%d\n", rspObj.rsp.blockNum); + while (++rspObj.resIter < rspObj.rsp.blockNum) { + SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(rspObj.rsp.blockData, rspObj.resIter); + if (!rspObj.rsp.withSchema) { + uError("WriteRaw:no schema, iter:%d", rspObj.resIter); + goto end; + } + SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.blockSchema, rspObj.resIter); + setResSchemaInfo(&rspObj.resInfo, pSW->pSchema, pSW->nCols); + + code = setQueryResultFromRsp(&rspObj.resInfo, pRetrieve, false, false); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw: setQueryResultFromRsp error"); + goto end; + } + + const char* tbName = (const char*)taosArrayGetP(rspObj.rsp.blockTbName, rspObj.resIter); + if (!tbName) { + uError("WriteRaw: tbname is null"); + code = TSDB_CODE_TMQ_INVALID_MSG; + goto end; + } + + printf("raw data tbname:%s\n", tbName); + SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}}; + strcpy(pName.dbname, pRequest->pDb); + strcpy(pName.tname, tbName); + + VgData vgData = {0}; + code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &(vgData.vg)); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:catalogGetTableHashVgroup failed. table name: %s", tbName); + goto end; + } + + // find schema data info + int32_t schemaLen = 0; + void* schemaData = NULL; + for(int j = 0; j < rspObj.rsp.createTableNum; j++){ + void** dataTmp = taosArrayGet(rspObj.rsp.createTableReq, j); + int32_t* lenTmp = taosArrayGet(rspObj.rsp.createTableLen, j); + + SDecoder decoderTmp = {0}; + SVCreateTbReq pCreateReq = {0}; + + tDecoderInit(&decoderTmp, *dataTmp, *lenTmp); + if (tDecodeSVCreateTbReq(&decoderTmp, &pCreateReq) < 0) { + tDecoderClear(&decoderTmp); + taosMemoryFreeClear(pCreateReq.comment); + taosArrayDestroy(pCreateReq.ctb.tagName); + goto end; + } + + ASSERT (pCreateReq.type == TSDB_CHILD_TABLE); + if(strcmp(tbName, pCreateReq.name) == 0){ + schemaLen = *lenTmp; + schemaData = *dataTmp; + strcpy(pName.tname, pCreateReq.ctb.name); + tDecoderClear(&decoderTmp); + taosMemoryFreeClear(pCreateReq.comment); + taosArrayDestroy(pCreateReq.ctb.tagName); + break; + } + tDecoderClear(&decoderTmp); + taosMemoryFreeClear(pCreateReq.comment); + taosArrayDestroy(pCreateReq.ctb.tagName); + } + + code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta); + if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { + uError("WriteRaw:catalogGetTableMeta table not exist. table name: %s", tbName); + code = TSDB_CODE_SUCCESS; + continue; + } + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:catalogGetTableMeta failed. table name: %s", tbName); + goto end; + } + + uint16_t fLen = 0; + int32_t rowSize = 0; + int16_t nVar = 0; + for (int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { + SSchema* schema = &pTableMeta->schema[i]; + fLen += TYPE_BYTES[schema->type]; + rowSize += schema->bytes; + if (IS_VAR_DATA_TYPE(schema->type)) { + nVar++; + } + } + + int32_t rows = rspObj.resInfo.numOfRows; + int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + + (int32_t)TD_BITMAP_BYTES(pTableMeta->tableInfo.numOfColumns - 1); + + int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize; + + SSubmitReq* subReq = NULL; + SSubmitBlk* blk = NULL; + void* hData = taosHashGet(pVgHash, &vgData.vg.vgId, sizeof(vgData.vg.vgId)); + if (hData) { + vgData = *(VgData*)hData; + + int32_t totalLen = ((SSubmitReq*)(vgData.data))->length + submitLen; + void* tmp = taosMemoryRealloc(vgData.data, totalLen); + if (tmp == NULL) { + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto end; + } + vgData.data = tmp; + ((VgData*)hData)->data = tmp; + subReq = (SSubmitReq*)(vgData.data); + blk = POINTER_SHIFT(vgData.data, subReq->length); + } else { + int32_t totalLen = sizeof(SSubmitReq) + submitLen; + void* tmp = taosMemoryCalloc(1, totalLen); + if (tmp == NULL) { + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto end; + } + vgData.data = tmp; + taosHashPut(pVgHash, (const char*)&vgData.vg.vgId, sizeof(vgData.vg.vgId), (char*)&vgData, sizeof(vgData)); + subReq = (SSubmitReq*)(vgData.data); + subReq->length = sizeof(SSubmitReq); + subReq->numOfBlocks = 0; + + blk = POINTER_SHIFT(vgData.data, sizeof(SSubmitReq)); + } + + // pSW->pSchema should be same as pTableMeta->schema + // ASSERT(pSW->nCols == pTableMeta->tableInfo.numOfColumns); + uint64_t suid = (TSDB_NORMAL_TABLE == pTableMeta->tableType ? 0 : pTableMeta->suid); + uint64_t uid = pTableMeta->uid; + int16_t sver = pTableMeta->sversion; + + void* blkSchema = POINTER_SHIFT(blk, sizeof(SSubmitBlk)); + if(schemaData){ + memcpy(blkSchema, schemaData, schemaLen); + } + STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen); + + SRowBuilder rb = {0}; + tdSRowInit(&rb, sver); + tdSRowSetTpInfo(&rb, pTableMeta->tableInfo.numOfColumns, fLen); + int32_t totalLen = 0; + + SHashObj* schemaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + for (int i = 0; i < pSW->nCols; i++) { + SSchema* schema = &pSW->pSchema[i]; + taosHashPut(schemaHash, schema->name, strlen(schema->name), &i, sizeof(int32_t)); + } + + for (int32_t j = 0; j < rows; j++) { + tdSRowResetBuf(&rb, rowData); + + doSetOneRowPtr(&rspObj.resInfo); + rspObj.resInfo.current += 1; + + int32_t offset = 0; + for (int32_t k = 0; k < pTableMeta->tableInfo.numOfColumns; k++) { + const SSchema* pColumn = &pTableMeta->schema[k]; + int32_t* index = taosHashGet(schemaHash, pColumn->name, strlen(pColumn->name)); + if (!index) { + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); + } else { + char* colData = rspObj.resInfo.row[*index]; + if (!colData) { + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); + } else { + if (IS_VAR_DATA_TYPE(pColumn->type)) { + colData -= VARSTR_HEADER_SIZE; + } + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, offset, k); + } + } + + offset += TYPE_BYTES[pColumn->type]; + } + tdSRowEnd(&rb); + int32_t rowLen = TD_ROW_LEN(rowData); + rowData = POINTER_SHIFT(rowData, rowLen); + totalLen += rowLen; + } + + taosHashCleanup(schemaHash); + blk->uid = htobe64(uid); + blk->suid = htobe64(suid); + blk->sversion = htonl(sver); + blk->schemaLen = htonl(schemaLen); + blk->numOfRows = htonl(rows); + blk->dataLen = htonl(totalLen); + subReq->length += sizeof(SSubmitBlk) + schemaLen + totalLen; + subReq->numOfBlocks++; + taosMemoryFreeClear(pTableMeta); + rspObj.resInfo.pRspMsg = NULL; + doFreeReqResultInfo(&rspObj.resInfo); + } + + pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); + if (NULL == pQuery) { + uError("create SQuery error"); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; + pQuery->haveResultSet = false; + pQuery->msgType = TDMT_VND_SUBMIT; + pQuery->pRoot = (SNode*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); + if (NULL == pQuery->pRoot) { + uError("create pQuery->pRoot error"); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + SVnodeModifOpStmt* nodeStmt = (SVnodeModifOpStmt*)(pQuery->pRoot); + nodeStmt->payloadType = PAYLOAD_TYPE_KV; + + int32_t numOfVg = taosHashGetSize(pVgHash); + nodeStmt->pDataBlocks = taosArrayInit(numOfVg, POINTER_BYTES); + + VgData* vData = (VgData*)taosHashIterate(pVgHash, NULL); + while (vData) { + SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); + if (NULL == dst) { + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto end; + } + dst->vg = vData->vg; + SSubmitReq* subReq = (SSubmitReq*)(vData->data); + dst->numOfTables = subReq->numOfBlocks; + dst->size = subReq->length; + dst->pData = (char*)subReq; + vData->data = NULL; // no need free + subReq->header.vgId = htonl(dst->vg.vgId); + subReq->version = htonl(1); + subReq->header.contLen = htonl(subReq->length); + subReq->length = htonl(subReq->length); + subReq->numOfBlocks = htonl(subReq->numOfBlocks); + taosArrayPush(nodeStmt->pDataBlocks, &dst); + vData = (VgData*)taosHashIterate(pVgHash, vData); + } + + launchQueryImpl(pRequest, pQuery, true, NULL); + code = pRequest->code; + + end: + tDeleteSTaosxRsp(&rspObj.rsp); + rspObj.resInfo.pRspMsg = NULL; + doFreeReqResultInfo(&rspObj.resInfo); tDecoderClear(&decoder); qDestroyQuery(pQuery); destroyRequest(pRequest); @@ -1587,10 +1965,15 @@ end: } char* tmq_get_json_meta(TAOS_RES* res) { - if (!TD_RES_TMQ_META(res)) { + if (!TD_RES_TMQ_META(res) && !TD_RES_TMQ_METADATA(res)) { return NULL; } + if(TD_RES_TMQ_METADATA(res)){ + SMqTaosxRspObj* pMetaDataRspObj = (SMqTaosxRspObj*)res; + return processAutoCreateTable(&pMetaDataRspObj->rsp); + } + SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_CREATE_STB) { return processCreateStb(&pMetaRspObj->metaRsp); @@ -1638,6 +2021,25 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) { raw->raw = buf; raw->raw_len = len; raw->raw_type = RES_TYPE__TMQ; + } else if (TD_RES_TMQ_METADATA(res)) { + SMqTaosxRspObj* rspObj = ((SMqTaosxRspObj*)res); + + int32_t len = 0; + int32_t code = 0; + tEncodeSize(tEncodeSTaosxRsp, &rspObj->rsp, len, code); + if (code < 0) { + return -1; + } + + void* buf = taosMemoryCalloc(1, len); + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, len); + tEncodeSTaosxRsp(&encoder, &rspObj->rsp); + tEncoderClear(&encoder); + + raw->raw = buf; + raw->raw_len = len; + raw->raw_type = RES_TYPE__TMQ_METADATA; } else { return TSDB_CODE_TMQ_INVALID_MSG; } @@ -1645,7 +2047,7 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) { } void tmq_free_raw(tmq_raw_data raw) { - if (raw.raw_type == RES_TYPE__TMQ) { + if (raw.raw_type == RES_TYPE__TMQ || raw.raw_type == RES_TYPE__TMQ_METADATA) { taosMemoryFree(raw.raw); } } @@ -1671,6 +2073,8 @@ int32_t tmq_write_raw(TAOS* taos, tmq_raw_data raw) { return taosDeleteData(taos, raw.raw, raw.raw_len); } else if (raw.raw_type == RES_TYPE__TMQ) { return tmqWriteRawDataImpl(taos, raw.raw, raw.raw_len); + } else if (raw.raw_type == RES_TYPE__TMQ_METADATA) { + return tmqWriteRawMetaDataImpl(taos, raw.raw, raw.raw_len); } return TSDB_CODE_INVALID_PARA; } diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index fc95bd86dc..c9c02a77e1 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -515,6 +515,10 @@ int32_t tmqCommitMsgImpl(tmq_t* tmq, const TAOS_RES* msg, int8_t async, tmq_comm SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)msg; topic = pMetaRspObj->topic; vgId = pMetaRspObj->vgId; + } else if(TD_RES_TMQ_METADATA(msg)) { + SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)msg; + topic = pRspObj->topic; + vgId = pRspObj->vgId; } else { return TSDB_CODE_TMQ_INVALID_MSG; } @@ -1471,16 +1475,16 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) { SMqTaosxRspObj* tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper) { SMqTaosxRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqTaosxRspObj)); - pRspObj->resType = RES_TYPE__TAOSX; + pRspObj->resType = RES_TYPE__TMQ_METADATA; tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN); pRspObj->vgId = pWrapper->vgHandle->vgId; pRspObj->resIter = -1; - memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqTaosxRspObj)); + memcpy(&pRspObj->rsp, &pWrapper->taosxRsp, sizeof(STaosxRsp)); pRspObj->resInfo.totalRows = 0; pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI; - if (!pWrapper->dataRsp.withSchema) { + if (!pWrapper->taosxRsp.withSchema) { setResSchemaInfo(&pRspObj->resInfo, pWrapper->topicHandle->schema.pSchema, pWrapper->topicHandle->schema.nCols); } @@ -1654,8 +1658,14 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { rspWrapper = NULL; continue; } + // build rsp - SMqRspObj* pRsp = tmqBuildRspFromWrapper(pollRspWrapper); + void* pRsp = NULL; + if(pollRspWrapper->taosxRsp.createTableNum == 0){ + pRsp = tmqBuildRspFromWrapper(pollRspWrapper); + }else{ + pRsp = tmqBuildTaosxRspFromWrapper(pollRspWrapper); + } taosFreeQitem(pollRspWrapper); return pRsp; } else { @@ -1775,11 +1785,11 @@ tmq_res_t tmq_get_res_type(TAOS_RES* res) { } else if (TD_RES_TMQ_META(res)) { SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_DELETE) { - return TMQ_RES_TAOSX; + return TMQ_RES_DATA; } return TMQ_RES_TABLE_META; - } else if (TD_RES_TMQ_TAOSX(res)) { - return TMQ_RES_DATA; + } else if (TD_RES_TMQ_METADATA(res)) { + return TMQ_RES_METADATA; } else { return TMQ_RES_INVALID; } @@ -1792,6 +1802,9 @@ const char* tmq_get_topic_name(TAOS_RES* res) { } else if (TD_RES_TMQ_META(res)) { SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; return strchr(pMetaRspObj->topic, '.') + 1; + } else if (TD_RES_TMQ_METADATA(res)) { + SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)res; + return strchr(pRspObj->topic, '.') + 1; } else { return NULL; } @@ -1804,6 +1817,9 @@ const char* tmq_get_db_name(TAOS_RES* res) { } else if (TD_RES_TMQ_META(res)) { SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; return strchr(pMetaRspObj->db, '.') + 1; + } else if (TD_RES_TMQ_METADATA(res)) { + SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)res; + return strchr(pRspObj->db, '.') + 1; } else { return NULL; } @@ -1816,6 +1832,9 @@ int32_t tmq_get_vgroup_id(TAOS_RES* res) { } else if (TD_RES_TMQ_META(res)) { SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; return pMetaRspObj->vgId; + } else if (TD_RES_TMQ_METADATA(res)) { + SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)res; + return pRspObj->vgId; } else { return -1; } @@ -1829,7 +1848,14 @@ const char* tmq_get_table_name(TAOS_RES* res) { return NULL; } return (const char*)taosArrayGetP(pRspObj->rsp.blockTbName, pRspObj->resIter); - } + } else if (TD_RES_TMQ_METADATA(res)) { + SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)res; + if (!pRspObj->rsp.withTbName || pRspObj->rsp.blockTbName == NULL || pRspObj->resIter < 0 || + pRspObj->resIter >= pRspObj->rsp.blockNum) { + return NULL; + } + return (const char*)taosArrayGetP(pRspObj->rsp.blockTbName, pRspObj->resIter); + } return NULL; } diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 16b8e55cf7..8a8e6a83a5 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1446,6 +1446,7 @@ size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) { int32_t payloadSize = pageSize - blockDataGetSerialMetaSize(numOfCols); int32_t rowSize = pBlock->info.rowSize; int32_t nRows = payloadSize / rowSize; + ASSERT(nRows >= 1); // the true value must be less than the value of nRows int32_t additional = 0; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index ddda8f8c9a..a6e30ae0b5 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -163,6 +163,7 @@ int32_t tsTtlUnit = 86400; int32_t tsTtlPushInterval = 86400; int32_t tsGrantHBInterval = 60; int32_t tsUptimeInterval = 300; // seconds +char tsUdfdResFuncs[1024] = ""; // udfd resident funcs that teardown when udfd exits #ifndef _STORAGE int32_t taosSetTfsCfg(SConfig *pCfg) { @@ -385,9 +386,9 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsNumOfQnodeQueryThreads = TMAX(tsNumOfQnodeQueryThreads, 4); if (cfgAddInt32(pCfg, "numOfQnodeQueryThreads", tsNumOfQnodeQueryThreads, 1, 1024, 0) != 0) return -1; -// tsNumOfQnodeFetchThreads = tsNumOfCores / 2; -// tsNumOfQnodeFetchThreads = TMAX(tsNumOfQnodeFetchThreads, 4); -// if (cfgAddInt32(pCfg, "numOfQnodeFetchThreads", tsNumOfQnodeFetchThreads, 1, 1024, 0) != 0) return -1; + // tsNumOfQnodeFetchThreads = tsNumOfCores / 2; + // tsNumOfQnodeFetchThreads = TMAX(tsNumOfQnodeFetchThreads, 4); + // if (cfgAddInt32(pCfg, "numOfQnodeFetchThreads", tsNumOfQnodeFetchThreads, 1, 1024, 0) != 0) return -1; tsNumOfSnodeSharedThreads = tsNumOfCores / 4; tsNumOfSnodeSharedThreads = TRANGE(tsNumOfSnodeSharedThreads, 2, 4); @@ -421,6 +422,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "uptimeInterval", tsUptimeInterval, 1, 100000, 1) != 0) return -1; if (cfgAddBool(pCfg, "udf", tsStartUdfd, 0) != 0) return -1; + if (cfgAddString(pCfg, "udfdResFuncs", tsUdfdResFuncs, 0) != 0) return -1; GRANT_CFG_ADD; return 0; } @@ -527,15 +529,15 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) { pItem->stype = stype; } -/* - pItem = cfgGetItem(tsCfg, "numOfQnodeFetchThreads"); - if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) { - tsNumOfQnodeFetchThreads = numOfCores / 2; - tsNumOfQnodeFetchThreads = TMAX(tsNumOfQnodeFetchThreads, 4); - pItem->i32 = tsNumOfQnodeFetchThreads; - pItem->stype = stype; - } -*/ + /* + pItem = cfgGetItem(tsCfg, "numOfQnodeFetchThreads"); + if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) { + tsNumOfQnodeFetchThreads = numOfCores / 2; + tsNumOfQnodeFetchThreads = TMAX(tsNumOfQnodeFetchThreads, 4); + pItem->i32 = tsNumOfQnodeFetchThreads; + pItem->stype = stype; + } + */ pItem = cfgGetItem(tsCfg, "numOfSnodeSharedThreads"); if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) { @@ -693,7 +695,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsNumOfVnodeSyncThreads = cfgGetItem(pCfg, "numOfVnodeSyncThreads")->i32; tsNumOfVnodeRsmaThreads = cfgGetItem(pCfg, "numOfVnodeRsmaThreads")->i32; tsNumOfQnodeQueryThreads = cfgGetItem(pCfg, "numOfQnodeQueryThreads")->i32; -// tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32; + // tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32; tsNumOfSnodeSharedThreads = cfgGetItem(pCfg, "numOfSnodeSharedThreads")->i32; tsNumOfSnodeUniqueThreads = cfgGetItem(pCfg, "numOfSnodeUniqueThreads")->i32; tsRpcQueueMemoryAllowed = cfgGetItem(pCfg, "rpcQueueMemoryAllowed")->i64; @@ -717,6 +719,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsUptimeInterval = cfgGetItem(pCfg, "uptimeInterval")->i32; tsStartUdfd = cfgGetItem(pCfg, "udf")->bval; + tstrncpy(tsUdfdResFuncs, cfgGetItem(pCfg, "udfdResFuncs")->str, sizeof(tsUdfdResFuncs)); if (tsQueryBufferSize >= 0) { tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; @@ -941,10 +944,10 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) { tsNumOfVnodeRsmaThreads = cfgGetItem(pCfg, "numOfVnodeRsmaThreads")->i32; } else if (strcasecmp("numOfQnodeQueryThreads", name) == 0) { tsNumOfQnodeQueryThreads = cfgGetItem(pCfg, "numOfQnodeQueryThreads")->i32; -/* - } else if (strcasecmp("numOfQnodeFetchThreads", name) == 0) { - tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32; -*/ + /* + } else if (strcasecmp("numOfQnodeFetchThreads", name) == 0) { + tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32; + */ } else if (strcasecmp("numOfSnodeSharedThreads", name) == 0) { tsNumOfSnodeSharedThreads = cfgGetItem(pCfg, "numOfSnodeSharedThreads")->i32; } else if (strcasecmp("numOfSnodeUniqueThreads", name) == 0) { diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index fffaf214ed..0990815024 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -6018,12 +6018,18 @@ int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, STaosxRsp *pRsp) { void tDeleteSTaosxRsp(STaosxRsp *pRsp) { taosArrayDestroy(pRsp->blockDataLen); + pRsp->blockDataLen = NULL; taosArrayDestroyP(pRsp->blockData, (FDelete)taosMemoryFree); + pRsp->blockData = NULL; taosArrayDestroyP(pRsp->blockSchema, (FDelete)tDeleteSSchemaWrapper); + pRsp->blockSchema = NULL; taosArrayDestroyP(pRsp->blockTbName, (FDelete)taosMemoryFree); + pRsp->blockTbName = NULL; taosArrayDestroy(pRsp->createTableLen); + pRsp->createTableLen = NULL; taosArrayDestroyP(pRsp->createTableReq, (FDelete)taosMemoryFree); + pRsp->createTableReq = NULL; } int32_t tEncodeSSingleDeleteReq(SEncoder *pEncoder, const SSingleDeleteReq *pReq) { diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index a9723abacc..537c9ca834 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -540,7 +540,7 @@ typedef struct { } SMqConsumerEp; SMqConsumerEp* tCloneSMqConsumerEp(const SMqConsumerEp* pEp); -void tDeleteSMqConsumerEp(SMqConsumerEp* pEp); +void tDeleteSMqConsumerEp(void* pEp); int32_t tEncodeSMqConsumerEp(void** buf, const SMqConsumerEp* pEp); void* tDecodeSMqConsumerEp(const void* buf, SMqConsumerEp* pEp); diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 4470a6b064..cdfa208665 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -197,11 +197,12 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { SMqConsumerLostMsg *pLostMsg = rpcMallocCont(sizeof(SMqConsumerLostMsg)); pLostMsg->consumerId = pConsumer->consumerId; - SRpcMsg *pRpcMsg = taosMemoryCalloc(1, sizeof(SRpcMsg)); - pRpcMsg->msgType = TDMT_MND_MQ_CONSUMER_LOST; - pRpcMsg->pCont = pLostMsg; - pRpcMsg->contLen = sizeof(SMqConsumerLostMsg); - tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, pRpcMsg); + SRpcMsg pRpcMsg = { + .msgType = TDMT_MND_MQ_CONSUMER_LOST, + .pCont = pLostMsg, + .contLen = sizeof(SMqConsumerLostMsg), + }; + tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &pRpcMsg); } if (status == MQ_CONSUMER_STATUS__LOST_REBD || status == MQ_CONSUMER_STATUS__READY) { // do nothing @@ -280,11 +281,12 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) { SMqConsumerRecoverMsg *pRecoverMsg = rpcMallocCont(sizeof(SMqConsumerRecoverMsg)); pRecoverMsg->consumerId = consumerId; - SRpcMsg *pRpcMsg = taosMemoryCalloc(1, sizeof(SRpcMsg)); - pRpcMsg->msgType = TDMT_MND_MQ_CONSUMER_RECOVER; - pRpcMsg->pCont = pRecoverMsg; - pRpcMsg->contLen = sizeof(SMqConsumerRecoverMsg); - tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, pRpcMsg); + SRpcMsg pRpcMsg = { + .msgType = TDMT_MND_MQ_CONSUMER_RECOVER, + .pCont = pRecoverMsg, + .contLen = sizeof(SMqConsumerRecoverMsg), + }; + tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &pRpcMsg); } mndReleaseConsumer(pMnode, pConsumer); @@ -318,11 +320,12 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { SMqConsumerRecoverMsg *pRecoverMsg = rpcMallocCont(sizeof(SMqConsumerRecoverMsg)); pRecoverMsg->consumerId = consumerId; - SRpcMsg *pRpcMsg = taosMemoryCalloc(1, sizeof(SRpcMsg)); - pRpcMsg->msgType = TDMT_MND_MQ_CONSUMER_RECOVER; - pRpcMsg->pCont = pRecoverMsg; - pRpcMsg->contLen = sizeof(SMqConsumerRecoverMsg); - tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, pRpcMsg); + SRpcMsg pRpcMsg = { + .msgType = TDMT_MND_MQ_CONSUMER_RECOVER, + .pCont = pRecoverMsg, + .contLen = sizeof(SMqConsumerRecoverMsg), + }; + tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &pRpcMsg); } #endif diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index f1280700f0..50d26ce9a5 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -338,8 +338,8 @@ SMqConsumerEp *tCloneSMqConsumerEp(const SMqConsumerEp *pConsumerEpOld) { return pConsumerEpNew; } -void tDeleteSMqConsumerEp(SMqConsumerEp *pConsumerEp) { - // +void tDeleteSMqConsumerEp(void *data) { + SMqConsumerEp *pConsumerEp = (SMqConsumerEp*)data; taosArrayDestroyP(pConsumerEp->vgs, (FDelete)tDeleteSMqVgEp); } diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 7a99d26683..1f7a059ffc 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -51,7 +51,6 @@ target_sources( "src/tsdb/tsdbCacheRead.c" "src/tsdb/tsdbRetention.c" "src/tsdb/tsdbDiskData.c" - "src/tsdb/tsdbCompress.c" "src/tsdb/tsdbCompact.c" "src/tsdb/tsdbMergeTree.c" diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index 9931462e5f..dade85b12d 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -146,6 +146,7 @@ struct SRSmaInfoItem { uint16_t nScanned; int32_t maxDelay; // ms tmr_h tmrId; + void *pStreamState; }; struct SRSmaInfo { @@ -224,8 +225,10 @@ int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type); int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash); int32_t tdRSmaProcessRestoreImpl(SSma *pSma, int8_t type, int64_t qtaskFileVer); -void tdRSmaQTaskInfoGetFileName(int32_t vid, int64_t version, char *outputName); -void tdRSmaQTaskInfoGetFullName(int32_t vid, int64_t version, const char *path, char *outputName); +void tdRSmaQTaskInfoGetFileName(int32_t vgId, int64_t version, char *outputName); +void tdRSmaQTaskInfoGetFullName(int32_t vgId, int64_t version, const char *path, char *outputName); +void tdRSmaQTaskInfoGetFullPath(int32_t vgId, int8_t level, const char *path, char *outputName); +void tdRSmaQTaskInfoGetFullPathEx(int32_t vgId, tb_uid_t suid, int8_t level, const char *path, char *outputName); static FORCE_INLINE void tdRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo) { int32_t ref = T_REF_INC(pRSmaInfo); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 0e85e7bfb6..4c8045d651 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -184,7 +184,7 @@ int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg); -SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchema* pSchema, bool createTb, int64_t suid, +SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchema* pSchema, SSchemaWrapper* pTagSchemaWrapper, bool createTb, int64_t suid, const char* stbFullName, SBatchDeleteReq* pDeleteReq); // sma diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index a34569b08e..24ade017d3 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -319,8 +319,12 @@ _query: pSchema = tCloneSSchemaWrapper(&meNew.stbEntry.schemaRow); tDecoderClear(&dcNew); tdbTbcClose(pCur); + tdbFree(pKey); + tdbFree(pVal); goto _exit; } + tdbFree(pKey); + tdbFree(pVal); tdbTbcClose(pCur); } } else if (me.type == TSDB_CHILD_TABLE) { @@ -347,11 +351,13 @@ _query: tDecoderClear(&dc); _exit: + tDecoderClear(&dc); metaULock(pMeta); tdbFree(pData); return pSchema; _err: + tDecoderClear(&dc); metaULock(pMeta); tdbFree(pData); return NULL; @@ -383,10 +389,8 @@ int metaTtlSmaller(SMeta *pMeta, uint64_t ttl, SArray *uidList) { ttlKey = *(STtlIdxKey *)pKey; taosArrayPush(uidList, &ttlKey.uid); } - tdbTbcClose(pCur); - tdbFree(pKey); - + tdbTbcClose(pCur); return 0; } diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c index 0edbd092e6..9fdbe50f88 100644 --- a/source/dnode/vnode/src/meta/metaSnapshot.c +++ b/source/dnode/vnode/src/meta/metaSnapshot.c @@ -353,6 +353,8 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t metaDebug("tmqsnap init idVersion uid:%" PRIi64 " version:%" PRIi64 " index:%d", *uid, idData->version, idData->index); } + tdbFree(pKey); + tdbFree(pVal); return TDB_CODE_SUCCESS; } @@ -528,6 +530,7 @@ int32_t getMetafromSnapShot(SSnapContext* ctx, void **pBuf, int32_t *contLen, in } } } + taosArrayDestroy(pTagVals); } // SIdInfo* sidInfo = (SIdInfo*)taosHashGet(ctx->idVersion, &me.ctbEntry.suid, sizeof(tb_uid_t)); // if(sidInfo->version >= idInfo->version){ diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 22ec8118a2..ce2a1c4b89 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -1192,10 +1192,11 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { const void *pTagData = NULL; // int32_t nTagData = 0; SDecoder dc = {0}; - + int32_t ret = 0; // get super table if (tdbTbGet(pMeta->pUidIdx, &pCtbEntry->ctbEntry.suid, sizeof(tb_uid_t), &pData, &nData) != 0) { - return -1; + ret = -1; + goto end; } tbDbKey.uid = pCtbEntry->ctbEntry.suid; tbDbKey.version = ((SUidIdxVal *)pData)[0].version; @@ -1221,17 +1222,20 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { // nTagData = ((const STag *)pCtbEntry->ctbEntry.pTags)->len; pTagData = pCtbEntry->ctbEntry.pTags; nTagData = ((const STag *)pCtbEntry->ctbEntry.pTags)->len; - return metaSaveJsonVarToIdx(pMeta, pCtbEntry, pTagColumn); + ret = metaSaveJsonVarToIdx(pMeta, pCtbEntry, pTagColumn); + goto end; } if (metaCreateTagIdxKey(pCtbEntry->ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, pCtbEntry->uid, &pTagIdxKey, &nTagIdxKey) < 0) { - return -1; + ret = -1; + goto end; } tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, &pMeta->txn); +end: metaDestroyTagIdxKey(pTagIdxKey); tDecoderClear(&dc); tdbFree(pData); - return 0; + return ret; } static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) { diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index ec8fcb2932..412def646f 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -92,6 +92,18 @@ void tdRSmaQTaskInfoGetFullName(int32_t vgId, int64_t version, const char *path, tdGetVndFileName(vgId, path, VNODE_RSMA_DIR, TD_QTASKINFO_FNAME_PREFIX, version, outputName); } +void tdRSmaQTaskInfoGetFullPath(int32_t vgId, int8_t level, const char *path, char *outputName) { + tdGetVndDirName(vgId, path, VNODE_RSMA_DIR, true, outputName); + int32_t rsmaLen = strlen(outputName); + snprintf(outputName + rsmaLen, TSDB_FILENAME_LEN - rsmaLen, "%" PRIi8, level); +} + +void tdRSmaQTaskInfoGetFullPathEx(int32_t vgId, tb_uid_t suid, int8_t level, const char *path, char *outputName) { + tdGetVndDirName(vgId, path, VNODE_RSMA_DIR, true, outputName); + int32_t rsmaLen = strlen(outputName); + snprintf(outputName + rsmaLen, TSDB_FILENAME_LEN - rsmaLen, "%" PRIi64 "%s%" PRIi8, suid, TD_DIRSEP, level); +} + static FORCE_INLINE int32_t tdRSmaQTaskInfoContLen(int32_t lenWithHead) { return lenWithHead - RSMA_QTASKINFO_HEAD_LEN; } @@ -130,6 +142,10 @@ void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree) { taosTmrStopA(&pItem->tmrId); } + if (isDeepFree && pItem->pStreamState) { + streamStateClose(pItem->pStreamState); + } + if (isDeepFree && pInfo->taskInfo[i]) { tdRSmaQTaskInfoFree(&pInfo->taskInfo[i], SMA_VID(pSma), i + 1); } else { @@ -290,12 +306,33 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat SRetention *pRetention = SMA_RETENTION(pSma); STsdbCfg *pTsdbCfg = SMA_TSDB_CFG(pSma); SVnode *pVnode = pSma->pVnode; + char taskInfDir[TSDB_FILENAME_LEN] = {0}; + void *pStreamState = NULL; + + // set the backend of stream state + tdRSmaQTaskInfoGetFullPathEx(TD_VID(pVnode), pRSmaInfo->suid, idx + 1, tfsGetPrimaryPath(pVnode->pTfs), taskInfDir); + if (!taosCheckExistFile(taskInfDir)) { + char *s = strdup(taskInfDir); + if (taosMulMkDir(taosDirName(s)) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + taosMemoryFree(s); + return TSDB_CODE_FAILED; + } + taosMemoryFree(s); + } + pStreamState = streamStateOpen(taskInfDir, NULL, true); + if (!pStreamState) { + terrno = TSDB_CODE_RSMA_STREAM_STATE_OPEN; + return TSDB_CODE_FAILED; + } + + SReadHandle handle = { .meta = pVnode->pMeta, .vnode = pVnode, .initTqReader = 1, + .pStateBackend = pStreamState, }; - pRSmaInfo->taskInfo[idx] = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle); if (!pRSmaInfo->taskInfo[idx]) { terrno = TSDB_CODE_RSMA_QTASKINFO_CREATE; @@ -303,6 +340,7 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat } SRSmaInfoItem *pItem = &(pRSmaInfo->items[idx]); pItem->triggerStat = TASK_TRIGGER_STAT_ACTIVE; // fetch the data when reboot + pItem->pStreamState = pStreamState; if (param->maxdelay[idx] < TSDB_MIN_ROLLUP_MAX_DELAY) { int64_t msInterval = convertTimeFromPrecisionToUnit(pRetention[idx + 1].freq, pTsdbCfg->precision, TIME_UNIT_MILLISECOND); @@ -322,7 +360,6 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat pItem->fetchLevel = pItem->level; taosTmrReset(tdRSmaFetchTrigger, RSMA_FETCH_INTERVAL, pItem, smaMgmt.tmrHandle, &pItem->tmrId); - smaInfo("vgId:%d, item:%p table:%" PRIi64 " level:%" PRIi8 " maxdelay:%" PRIi64 " watermark:%" PRIi64 ", finally maxdelay:%" PRIi32, @@ -1226,16 +1263,17 @@ int32_t tdRSmaProcessRestoreImpl(SSma *pSma, int8_t type, int64_t qtaskFileVer) if (tdRSmaRestoreQTaskInfoInit(pSma, &nTables) < 0) { goto _err; } - if (nTables <= 0) { smaDebug("vgId:%d, no need to restore rsma task %" PRIi8 " since no tables", SMA_VID(pSma), type); return TSDB_CODE_SUCCESS; } +#if 0 // step 2: retrieve qtaskinfo items from the persistence file(rsma/qtaskinfo) and restore if (tdRSmaRestoreQTaskInfoReload(pSma, type, qtaskFileVer) < 0) { goto _err; } +#endif // step 3: reload ts data from checkpoint if (tdRSmaRestoreTSDataReload(pSma) < 0) { @@ -1440,6 +1478,50 @@ static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, int8_t type, SRSmaQTaskInfoIte return TSDB_CODE_SUCCESS; } +int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { + SSma *pSma = pRSmaStat->pSma; + SVnode *pVnode = pSma->pVnode; + int32_t vid = SMA_VID(pSma); + + if (taosHashGetSize(pInfoHash) <= 0) { + return TSDB_CODE_SUCCESS; + } + + int64_t fsMaxVer = tdRSmaFSMaxVer(pSma, pRSmaStat); + if (pRSmaStat->commitAppliedVer <= fsMaxVer) { + smaDebug("vgId:%d, rsma persist, no need as applied %" PRIi64 " not larger than fsMaxVer %" PRIi64, vid, + pRSmaStat->commitAppliedVer, fsMaxVer); + return TSDB_CODE_SUCCESS; + } + + void *infoHash = NULL; + while ((infoHash = taosHashIterate(pInfoHash, infoHash))) { + SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash; + + if (RSMA_INFO_IS_DEL(pRSmaInfo)) { + continue; + } + + for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { + SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pRSmaInfo, i); + if (pItem && pItem->pStreamState) { + if (streamStateCommit(pItem->pStreamState) < 0) { + terrno = TSDB_CODE_RSMA_STREAM_STATE_COMMIT; + goto _err; + } + smaDebug("vgId:%d, rsma persist, stream state commit success, table %" PRIi64 " level %d", vid, pRSmaInfo->suid, + i + 1); + } + } + } + + return TSDB_CODE_SUCCESS; +_err: + smaError("vgId:%d, rsma persist failed since %s", vid, terrstr()); + return TSDB_CODE_FAILED; +} + +#if 0 int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { SSma *pSma = pRSmaStat->pSma; SVnode *pVnode = pSma->pVnode; @@ -1459,7 +1541,7 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { int64_t fsMaxVer = tdRSmaFSMaxVer(pSma, pRSmaStat); if (pRSmaStat->commitAppliedVer <= fsMaxVer) { smaDebug("vgId:%d, rsma persist, no need as applied %" PRIi64 " not larger than fsMaxVer %" PRIi64, vid, - pRSmaStat->commitAppliedVer, fsMaxVer); + pRSmaStat->commitAppliedVer, fsMaxVer); return TSDB_CODE_SUCCESS; } @@ -1579,6 +1661,8 @@ _err: return TSDB_CODE_FAILED; } +#endif + /** * @brief trigger to get rsma result in async mode * @@ -1926,7 +2010,7 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { if ((pEnv->flag & SMA_ENV_FLG_CLOSE) && (atomic_load_64(&pRSmaStat->nBufItems) <= 0)) { smaDebug("vgId:%d, exec task end, flag:%" PRIi8 ", nBufItems:%" PRIi64, SMA_VID(pSma), pEnv->flag, - atomic_load_64(&pRSmaStat->nBufItems)); + atomic_load_64(&pRSmaStat->nBufItems)); break; } } diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index e2cb51f586..6c32fbbc84 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -204,7 +204,7 @@ static int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char } SBatchDeleteReq deleteReq; - SSubmitReq *pSubmitReq = tqBlockToSubmit(pSma->pVnode, (const SArray *)msg, pTsmaStat->pTSchema, true, + SSubmitReq *pSubmitReq = tqBlockToSubmit(pSma->pVnode, (const SArray *)msg, pTsmaStat->pTSchema, &pTsmaStat->pTSma->schemaTag, true, pTsmaStat->pTSma->dstTbUid, pTsmaStat->pTSma->dstTbName, &deleteReq); if (!pSubmitReq) { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index c8841e5e16..98b7dd7163 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -51,6 +51,20 @@ void tqCleanUp() { } } +static void destroySTqHandle(void* data) { + STqHandle* pData = (STqHandle*)data; + qDestroyTask(pData->execHandle.task); + if (pData->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { + } else if (pData->execHandle.subType == TOPIC_SUB_TYPE__DB) { + tqCloseReader(pData->execHandle.pExecReader); + walCloseReader(pData->pWalReader); + taosHashCleanup(pData->execHandle.execDb.pFilterOutTbUid); + } else if (pData->execHandle.subType == TOPIC_SUB_TYPE__TABLE){ + walCloseReader(pData->pWalReader); + tqCloseReader(pData->execHandle.pExecReader); + } +} + STQ* tqOpen(const char* path, SVnode* pVnode) { STQ* pTq = taosMemoryCalloc(1, sizeof(STQ)); if (pTq == NULL) { @@ -62,6 +76,8 @@ STQ* tqOpen(const char* path, SVnode* pVnode) { pTq->pHandle = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK); + taosHashSetFreeFp(pTq->pHandle, destroySTqHandle); + pTq->pPushMgr = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); pTq->pCheckInfo = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK); @@ -141,11 +157,8 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con ASSERT(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum); ASSERT(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum); - if (pRsp->withSchema) { - ASSERT(taosArrayGetSize(pRsp->blockSchema) == pRsp->blockNum); - } else { - ASSERT(taosArrayGetSize(pRsp->blockSchema) == 0); - } + ASSERT(!pRsp->withSchema); + ASSERT(taosArrayGetSize(pRsp->blockSchema) == 0); if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) { if (pRsp->blockNum > 0) { @@ -520,6 +533,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { int64_t fetchVer = fetchOffsetNew.version + 1; pCkHead = taosMemoryMalloc(sizeof(SWalCkHead) + 2048); if (pCkHead == NULL) { + tDeleteSTaosxRsp(&taosxRsp); return -1; } @@ -580,14 +594,17 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { if (tqSendMetaPollRsp(pTq, pMsg, pReq, &metaRsp) < 0) { code = -1; taosMemoryFree(pCkHead); + tDeleteSTaosxRsp(&taosxRsp); return code; } code = 0; if (pCkHead) taosMemoryFree(pCkHead); + tDeleteSTaosxRsp(&taosxRsp); return code; } } } + tDeleteSTaosxRsp(&taosxRsp); return 0; } @@ -760,7 +777,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask) { // expand executor if (pTask->taskLevel == TASK_LEVEL__SOURCE) { - pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pTask); + pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pTask, false); if (pTask->pState == NULL) { return -1; } @@ -774,7 +791,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask) { pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle); ASSERT(pTask->exec.executor); } else if (pTask->taskLevel == TASK_LEVEL__AGG) { - pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pTask); + pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pTask, false); if (pTask->pState == NULL) { return -1; } diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index d00907f677..a24f920235 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -243,14 +243,15 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp } if (pHandle->fetchMeta) { SSubmitBlk* pBlk = pReader->pBlock; - if (pBlk->schemaLen > 0) { + int32_t schemaLen = htonl(pBlk->schemaLen); + if (schemaLen > 0) { if (pRsp->createTableNum == 0) { pRsp->createTableLen = taosArrayInit(0, sizeof(int32_t)); pRsp->createTableReq = taosArrayInit(0, sizeof(void*)); } - void* createReq = taosMemoryCalloc(1, pBlk->schemaLen); - memcpy(createReq, pBlk->data, pBlk->schemaLen); - taosArrayPush(pRsp->createTableLen, &pBlk->schemaLen); + void* createReq = taosMemoryCalloc(1, schemaLen); + memcpy(createReq, pBlk->data, schemaLen); + taosArrayPush(pRsp->createTableLen, &schemaLen); taosArrayPush(pRsp->createTableReq, &createReq); pRsp->createTableNum++; } @@ -277,14 +278,15 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp } if (pHandle->fetchMeta) { SSubmitBlk* pBlk = pReader->pBlock; - if (pBlk->schemaLen > 0) { + int32_t schemaLen = htonl(pBlk->schemaLen); + if (schemaLen > 0) { if (pRsp->createTableNum == 0) { pRsp->createTableLen = taosArrayInit(0, sizeof(int32_t)); pRsp->createTableReq = taosArrayInit(0, sizeof(void*)); } - void* createReq = taosMemoryCalloc(1, pBlk->schemaLen); - memcpy(createReq, pBlk->data, pBlk->schemaLen); - taosArrayPush(pRsp->createTableLen, &pBlk->schemaLen); + void* createReq = taosMemoryCalloc(1, schemaLen); + memcpy(createReq, pBlk->data, schemaLen); + taosArrayPush(pRsp->createTableLen, &schemaLen); taosArrayPush(pRsp->createTableReq, &createReq); pRsp->createTableNum++; } diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index d8d8025151..7097591c35 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -145,6 +145,7 @@ int32_t tqOffsetCommitFile(STqOffsetStore* pStore) { ASSERT(0); tqError("write offset incomplete, len %d, write len %" PRId64, bodyLen, writeLen); taosHashCancelIterate(pStore->pHash, pIter); + taosMemoryFree(buf); return -1; } taosMemoryFree(buf); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 375130fa2c..8e5f328efa 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -80,6 +80,13 @@ bool isValValidForTable(STqHandle* pHandle, SWalCont *pHead){ void* buf = taosMemoryMalloc(tlen); if (NULL == buf) { taosArrayDestroy(reqNew.pArray); + for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { + pCreateReq = req.pReqs + iReq; + taosMemoryFreeClear(pCreateReq->comment); + if (pCreateReq->type == TSDB_CHILD_TABLE) { + taosArrayDestroy(pCreateReq->ctb.tagName); + } + } goto end; } SEncoder coderNew = {0}; @@ -91,6 +98,14 @@ bool isValValidForTable(STqHandle* pHandle, SWalCont *pHead){ taosMemoryFree(buf); taosArrayDestroy(reqNew.pArray); } + + for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { + pCreateReq = req.pReqs + iReq; + taosMemoryFreeClear(pCreateReq->comment); + if (pCreateReq->type == TSDB_CHILD_TABLE) { + taosArrayDestroy(pCreateReq->ctb.tagName); + } + } } else if (msgType == TDMT_VND_ALTER_TABLE) { SVAlterTbReq req = {0}; diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 522bf46aa1..16c2a5d033 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -48,7 +48,7 @@ int32_t tqBuildDeleteReq(SVnode* pVnode, const char* stbFullName, const SSDataBl return 0; } -SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchema* pTSchema, bool createTb, +SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchema* pTSchema, SSchemaWrapper* pTagSchemaWrapper, bool createTb, int64_t suid, const char* stbFullName, SBatchDeleteReq* pDeleteReq) { SSubmitReq* ret = NULL; SArray* schemaReqs = NULL; @@ -89,6 +89,32 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem return NULL; } + SArray *tagName = taosArrayInit(1, TSDB_COL_NAME_LEN); + char tagNameStr[TSDB_COL_NAME_LEN] = {0}; + strcpy(tagNameStr, "group_id"); + taosArrayPush(tagName, tagNameStr); + +// STag* pTag = NULL; +// taosArrayClear(tagArray); +// SArray *tagName = taosArrayInit(1, TSDB_COL_NAME_LEN); +// for(int j = 0; j < pTagSchemaWrapper->nCols; j++){ +// STagVal tagVal = { +// .cid = pTagSchemaWrapper->pSchema[j].colId, +// .type = pTagSchemaWrapper->pSchema[j].type, +// .i64 = (int64_t)pDataBlock->info.groupId, +// }; +// taosArrayPush(tagArray, &tagVal); +// taosArrayPush(tagName, pTagSchemaWrapper->pSchema[j].name); +// } +// +// tTagNew(tagArray, 1, false, &pTag); +// if (pTag == NULL) { +// terrno = TSDB_CODE_OUT_OF_MEMORY; +// taosArrayDestroy(tagArray); +// taosArrayDestroy(tagName); +// return NULL; +// } + SVCreateTbReq createTbReq = {0}; SName name = {0}; tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); @@ -99,6 +125,8 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem createTbReq.type = TSDB_CHILD_TABLE; createTbReq.ctb.suid = suid; createTbReq.ctb.pTag = (uint8_t*)pTag; + createTbReq.ctb.tagNum = taosArrayGetSize(tagArray); + createTbReq.ctb.tagName = tagName; int32_t code; int32_t schemaLen; @@ -113,6 +141,7 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem void* schemaStr = taosMemoryMalloc(schemaLen); if (schemaStr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; + tdDestroySVCreateTbReq(&createTbReq); return NULL; } taosArrayPush(schemaReqs, &schemaStr); @@ -123,6 +152,7 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem code = tEncodeSVCreateTbReq(&encoder, &createTbReq); if (code < 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; + tdDestroySVCreateTbReq(&createTbReq); return NULL; } tEncoderClear(&encoder); @@ -231,7 +261,7 @@ void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { ASSERT(pTask->tbSink.pTSchema); deleteReq.deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq)); - SSubmitReq* submitReq = tqBlockToSubmit(pVnode, pRes, pTask->tbSink.pTSchema, true, pTask->tbSink.stbUid, + SSubmitReq* submitReq = tqBlockToSubmit(pVnode, pRes, pTask->tbSink.pTSchema, pTask->tbSink.pSchemaWrapper, true, pTask->tbSink.stbUid, pTask->tbSink.stbFullName, &deleteReq); tqDebug("vgId:%d, task %d convert blocks over, put into write-queue", TD_VID(pVnode), pTask->taskId); @@ -274,7 +304,6 @@ void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { }; if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { - rpcFreeCont(submitReq); tqDebug("failed to put into write-queue since %s", terrstr()); } } diff --git a/source/dnode/vnode/src/tsdb/tsdbCompress.c b/source/dnode/vnode/src/tsdb/tsdbCompress.c deleted file mode 100644 index 76be7c1070..0000000000 --- a/source/dnode/vnode/src/tsdb/tsdbCompress.c +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tsdb.h" - -// Integer ===================================================== -typedef struct { - int8_t rawCopy; - int64_t prevVal; - int32_t nVal; - int32_t nBuf; - uint8_t *pBuf; -} SIntCompressor; - -#define I64_SAFE_ADD(a, b) (((a) >= 0 && (b) <= INT64_MAX - (b)) || ((a) < 0 && (b) >= INT64_MIN - (a))) -#define SIMPLE8B_MAX ((uint64_t)1152921504606846974LL) - -static int32_t tsdbCmprI64(SIntCompressor *pCompressor, int64_t val) { - int32_t code = 0; - - // raw copy - if (pCompressor->rawCopy) { - memcpy(pCompressor->pBuf + pCompressor->nBuf, &val, sizeof(val)); - pCompressor->nBuf += sizeof(val); - pCompressor->nVal++; - goto _exit; - } - - if (!I64_SAFE_ADD(val, pCompressor->prevVal)) { - pCompressor->rawCopy = 1; - // TODO: decompress and copy - pCompressor->nVal++; - goto _exit; - } - - int64_t diff = val - pCompressor->prevVal; - uint8_t zigzag = ZIGZAGE(int64_t, diff); - - if (zigzag >= SIMPLE8B_MAX) { - pCompressor->rawCopy = 1; - // TODO: decompress and copy - pCompressor->nVal++; - goto _exit; - } - -_exit: - return code; -} - -// Timestamp ===================================================== - -// Float ===================================================== \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 45fe29f0fa..5f03a82bc0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -320,6 +320,7 @@ void tLDataIterNextBlock(SLDataIter *pIter) { pIter->pSttBlk = NULL; if (index != -1) { + pIter->iSttBlk = index; pIter->pSttBlk = (SSttBlk *)taosArrayGet(pIter->pBlockLoadInfo->aSttBlk, pIter->iSttBlk); } } diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 18f6d3ad2c..6484b6cb31 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -462,6 +462,7 @@ typedef struct SPartitionDataInfo { typedef struct STimeWindowAggSupp { int8_t calTrigger; int64_t waterMark; + int64_t deleteMark; TSKEY maxTs; TSKEY minTs; SColumnInfoData timeWindowData; // query time window info for scalar function execution. @@ -1063,6 +1064,7 @@ bool functionNeedToExecute(SqlFunctionCtx* pCtx); bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup); bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup); bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup); +bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SOperatorInfo* pOperator, STimeWindowAggSupp* pTwSup); void appendOneRow(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, uint64_t* pGp); void printDataBlock(SSDataBlock* pBlock, const char* flag); uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId); @@ -1090,7 +1092,7 @@ int32_t setOutputBuf(STimeWindow* win, SResultRow** pResult, int64_t tableGroupI int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup, SExecTaskInfo* pTaskInfo); int32_t releaseOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult); -int32_t saveOutput(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult, int32_t resSize); +int32_t saveOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult, int32_t resSize); #ifdef __cplusplus } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index d75f0580e1..b26603b394 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -989,7 +989,8 @@ SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod if (pNode->output) { (*numOfOutputCols) += 1; - } else { + } else if (info != NULL) { + // select distinct tbname from stb where tbname='abc'; info->output = false; } } @@ -1341,7 +1342,7 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi return TSDB_CODE_SUCCESS; } -void cleanupQueryTableDataCond(SQueryTableDataCond* pCond) { taosMemoryFree(pCond->colList); } +void cleanupQueryTableDataCond(SQueryTableDataCond* pCond) { taosMemoryFreeClear(pCond->colList); } int32_t convertFillType(int32_t mode) { int32_t type = TSDB_FILL_NONE; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 44541ee27c..45b88ec6a5 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -876,6 +876,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pTaskInfo->tableqinfoList.pTableList, &pInfo->dataReader, NULL); + cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); strcpy(pTaskInfo->streamInfo.tbName, mtInfo.tbName); tDeleteSSchemaWrapper(pTaskInfo->streamInfo.schema); pTaskInfo->streamInfo.schema = mtInfo.schema; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index e0d417093c..a1ff79bd03 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3350,6 +3350,10 @@ static void cleanupTableSchemaInfo(SSchemaInfo* pSchemaInfo) { tDeleteSSchemaWrapper(pSchemaInfo->qsw); } +static void cleanupStreamInfo(SStreamTaskInfo* pStreamInfo) { + tDeleteSSchemaWrapper(pStreamInfo->schema); +} + static int32_t sortTableGroup(STableListInfo* pTableListInfo) { taosArrayClear(pTableListInfo->pGroupList); SArray* sortSupport = taosArrayInit(16, sizeof(uint64_t)); @@ -4043,6 +4047,7 @@ void doDestroyTask(SExecTaskInfo* pTaskInfo) { doDestroyTableList(&pTaskInfo->tableqinfoList); destroyOperatorInfo(pTaskInfo->pRoot); cleanupTableSchemaInfo(&pTaskInfo->schemaInfo); + cleanupStreamInfo(&pTaskInfo->streamInfo); nodesDestroyNode((SNode*)pTaskInfo->pSubplan); @@ -4166,9 +4171,8 @@ int32_t setOutputBuf(STimeWindow* win, SResultRow** pResult, int64_t tableGroupI }; char* value = NULL; int32_t size = pAggSup->resultRowSize; - /*if (streamStateGet(pTaskInfo->streamInfo.pState, &key, (void**)&value, &size) < 0) {*/ - /*value = taosMemoryCalloc(1, size);*/ - /*}*/ + + tSimpleHashPut(pAggSup->pResultRowHashTable, &key, sizeof(SWinKey), NULL, 0); if (streamStateAddIfNotExist(pTaskInfo->streamInfo.pState, &key, (void**)&value, &size) < 0) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -4186,7 +4190,7 @@ int32_t releaseOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pR return TSDB_CODE_SUCCESS; } -int32_t saveOutput(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult, int32_t resSize) { +int32_t saveOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult, int32_t resSize) { streamStatePut(pTaskInfo->streamInfo.pState, pKey, pResult, resSize); return TSDB_CODE_SUCCESS; } @@ -4259,8 +4263,9 @@ int32_t buildDataBlockFromGroupRes(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock } } } - releaseOutputBuf(pTaskInfo, &key, pRow); + pBlock->info.rows += pRow->numOfRows; + releaseOutputBuf(pTaskInfo, &key, pRow); } blockDataUpdateTsWindow(pBlock, 0); return TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index edc7ddb92e..8f9a78db83 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1331,7 +1331,7 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock // must check update info first. bool update = updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.uid, tsCol[rowId]); bool closedWin = isClosed && isSignleIntervalWindow(pInfo) && - isDeletedWindow(&win, pBlock->info.groupId, pInfo->windowSup.pIntervalAggSup); + isDeletedStreamWindow(&win, pBlock->info.groupId, pInfo->pTableScanOp, &pInfo->twAggSup); if ((update || closedWin) && out) { qDebug("stream update check not pass, update %d, closedWin %d", update, closedWin); uint64_t gpId = closedWin && pInfo->partitionSup.needCalc @@ -1780,6 +1780,7 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { qDebug("tmqsnap change get data uid:%ld", mtInfo.uid); qStreamPrepareScan(pTaskInfo, &pTaskInfo->streamInfo.prepareStatus, pInfo->sContext->subType); } + tDeleteSSchemaWrapper(mtInfo.schema); qDebug("tmqsnap stream scan tsdb return null"); return NULL; } else if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_META) { @@ -1931,11 +1932,6 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->pTagCond = pTagCond; pInfo->pGroupTags = pTableScanNode->pGroupTags; - pInfo->twAggSup = (STimeWindowAggSupp){ - .waterMark = pTableScanNode->watermark, - .calTrigger = pTableScanNode->triggerType, - .maxTs = INT64_MIN, - }; int32_t numOfCols = 0; pInfo->pColMatchInfo = extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID); @@ -1985,7 +1981,6 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->pUpdateInfo = NULL; pInfo->pTableScanOp = pTableScanOp; - pInfo->interval = pTSInfo->pdInfo.interval; pInfo->readHandle = *pHandle; pInfo->tableUid = pScanPhyNode->uid; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index c221aaf4fd..aed4ae9b26 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -867,6 +867,10 @@ static int32_t saveWinResultRow(SResultRow* result, uint64_t groupId, SHashObj* return saveWinResult(result->win.skey, result->pageId, result->offset, groupId, pUpdatedMap); } +static int32_t saveWinResultInfo(TSKEY ts, uint64_t groupId, SHashObj* pUpdatedMap) { + return saveWinResult(ts, -1, -1, groupId, pUpdatedMap); +} + static int32_t saveResultRow(SResultRow* result, uint64_t groupId, SArray* pUpdated) { return saveResult(result->win.skey, result->pageId, result->offset, groupId, pUpdated); } @@ -918,12 +922,16 @@ static void removeDeleteResults(SHashObj* pUpdatedMap, SArray* pDelWins) { } } -bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup) { - ASSERT(pSup->maxTs == INT64_MIN || pSup->maxTs > 0); - return pSup->maxTs != INT64_MIN && ts < pSup->maxTs - pSup->waterMark; +bool isOverdue(TSKEY ekey, STimeWindowAggSupp* pTwSup) { + ASSERT(pTwSup->maxTs == INT64_MIN || pTwSup->maxTs > 0); + return pTwSup->maxTs != INT64_MIN && ekey < pTwSup->maxTs - pTwSup->waterMark; } -bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup) { return isOverdue(pWin->ekey, pSup); } +bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pTwSup) { return isOverdue(pWin->ekey, pTwSup); } + +bool needDeleteWindowBuf(STimeWindow* pWin, STimeWindowAggSupp* pTwSup) { + return pTwSup->maxTs != INT64_MIN && pWin->ekey < pTwSup->maxTs - pTwSup->deleteMark; +} static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock, int32_t scanFlag) { @@ -1374,6 +1382,41 @@ static bool doClearWindow(SAggSupporter* pAggSup, SExprSupp* pSup, char* pData, return true; } +static bool doDeleteWindow(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, int32_t numOfOutput) { + SStreamIntervalOperatorInfo* pInfo = pOperator->info; + SWinKey key = {.ts = ts, .groupId = groupId}; + tSimpleHashRemove(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey)); + streamStateDel(pOperator->pTaskInfo->streamInfo.pState, &key); + return true; +} + +static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, int32_t numOfOutput, SSDataBlock* pBlock, + SArray* pUpWins, SHashObj* pUpdatedMap) { + SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); + TSKEY* startTsCols = (TSKEY*)pStartTsCol->pData; + SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); + TSKEY* endTsCols = (TSKEY*)pEndTsCol->pData; + SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); + uint64_t* pGpDatas = (uint64_t*)pGpCol->pData; + for (int32_t i = 0; i < pBlock->info.rows; i++) { + SResultRowInfo dumyInfo; + dumyInfo.cur.pageId = -1; + STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCols[i], pInterval, TSDB_ORDER_ASC); + while (win.ekey <= endTsCols[i]) { + uint64_t winGpId = pGpDatas[i]; + bool res = doDeleteWindow(pOperator, win.skey, winGpId, numOfOutput); + SWinKey winRes = {.ts = win.skey, .groupId = winGpId}; + if (pUpWins && res) { + taosArrayPush(pUpWins, &winRes); + } + if (pUpdatedMap) { + taosHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey)); + } + getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win); + } + } +} + bool doDeleteIntervalWindow(SAggSupporter* pAggSup, TSKEY ts, uint64_t groupId) { size_t bytes = sizeof(TSKEY); SET_RES_WINDOW_KEY(pAggSup->keyBuf, &ts, bytes, groupId); @@ -1383,8 +1426,6 @@ bool doDeleteIntervalWindow(SAggSupporter* pAggSup, TSKEY ts, uint64_t groupId) // window has been closed return false; } - // SFilePage* bufPage = getBufPage(pAggSup->pResultBuf, p1->pageId); - // dBufSetBufPageRecycled(pAggSup->pResultBuf, bufPage); tSimpleHashRemove(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); return true; } @@ -1512,6 +1553,49 @@ static int32_t closeIntervalWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pSup return TSDB_CODE_SUCCESS; } +static int32_t closeStreamIntervalWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SInterval* pInterval, + SHashObj* pPullDataMap, SHashObj* closeWins, SOperatorInfo* pOperator) { + qDebug("===stream===close interval window"); + void* pIte = NULL; + size_t keyLen = 0; + int32_t iter = 0; + while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) { + void* key = tSimpleHashGetKey(pIte, &keyLen); + SWinKey* pWinKey = (SWinKey*)key; + void* chIds = taosHashGet(pPullDataMap, pWinKey, sizeof(SWinKey)); + STimeWindow win = { + .skey = pWinKey->ts, + .ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1, + }; + if (isCloseWindow(&win, pTwSup)) { + if (chIds && pPullDataMap) { + SArray* chAy = *(SArray**)chIds; + int32_t size = taosArrayGetSize(chAy); + qDebug("===stream===window %" PRId64 " wait child size:%d", pWinKey->ts, size); + for (int32_t i = 0; i < size; i++) { + qDebug("===stream===window %" PRId64 " wait child id:%d", pWinKey->ts, *(int32_t*)taosArrayGet(chAy, i)); + } + continue; + } else if (pPullDataMap) { + qDebug("===stream===close window %" PRId64, pWinKey->ts); + } + + if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { + int32_t code = saveWinResultInfo(pWinKey->ts, pWinKey->groupId, closeWins); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + tSimpleHashIterateRemove(pHashMap, pWinKey, sizeof(SWinKey), &pIte, &iter); + + if (needDeleteWindowBuf(&win, pTwSup)) { + streamStateDel(pOperator->pTaskInfo->streamInfo.pState, pWinKey); + } + } + } + return TSDB_CODE_SUCCESS; +} + static void closeChildIntervalWindow(SArray* pChildren, TSKEY maxTs) { int32_t size = taosArrayGetSize(pChildren); for (int32_t i = 0; i < size; i++) { @@ -1669,16 +1753,17 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SInt } void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SAggSupporter* pSup, SInterval* pInterval, - int64_t waterMark) { + STimeWindowAggSupp* pTwSup) { if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { - initIntervalDownStream(downstream->pDownstream[0], type, pSup, pInterval, waterMark); + initIntervalDownStream(downstream->pDownstream[0], type, pSup, pInterval, pTwSup); return; } SStreamScanInfo* pScanInfo = downstream->info; pScanInfo->windowSup.parentType = type; pScanInfo->windowSup.pIntervalAggSup = pSup; - pScanInfo->pUpdateInfo = updateInfoInitP(pInterval, waterMark); + pScanInfo->pUpdateInfo = updateInfoInitP(pInterval, pTwSup->waterMark); pScanInfo->interval = *pInterval; + pScanInfo->twAggSup = *pTwSup; } void initStreamFunciton(SqlFunctionCtx* pCtx, int32_t numOfExpr) { @@ -1763,11 +1848,6 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, NULL, NULL, destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); - if (nodeType(pPhyNode) == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL) { - initIntervalDownStream(downstream, QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, &pInfo->aggSup, &pInfo->interval, - pInfo->twAggSup.waterMark); - } - code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -2784,6 +2864,19 @@ bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup) { return p1 == NULL; } +bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SOperatorInfo* pOperator, STimeWindowAggSupp* pTwSup) { + if (pWin->ekey < pTwSup->maxTs - pTwSup->deleteMark) { + SWinKey key = {.ts = pWin->skey, .groupId = groupId}; + void* pVal = NULL; + int32_t size = 0; + if (streamStateGet(pOperator->pTaskInfo->streamInfo.pState, &key, &pVal, &size) < 0) { + return false; + } + streamStateReleaseBuf(pOperator->pTaskInfo->streamInfo.pState, &key, pVal); + } + return false; +} + int32_t getNexWindowPos(SInterval* pInterval, SDataBlockInfo* pBlockInfo, TSKEY* tsCols, int32_t startPos, TSKEY eKey, STimeWindow* pNextWin) { int32_t forwardRows = @@ -3341,7 +3434,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, NULL, destroyStreamFinalIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL) { - initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, pInfo->twAggSup.waterMark); + initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, &pInfo->twAggSup); } code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -4918,8 +5011,8 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR SExprSupp* pSup = &pOperatorInfo->exprSupp; SInterval* pInterval = &iaInfo->interval; - int32_t startPos = 0; - int64_t* tsCols = extractTsCol(pBlock, iaInfo); + int32_t startPos = 0; + int64_t* tsCols = extractTsCol(pBlock, iaInfo); TSKEY ts = getStartTsKey(&pBlock->info.window, tsCols); @@ -4938,7 +5031,7 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR win.skey = miaInfo->curTs; win.ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; - int32_t ret = setSingleOutputTupleBuf(pResultRowInfo, &win, &miaInfo->pResultRow, pSup, &iaInfo->aggSup); + int32_t ret = setSingleOutputTupleBuf(pResultRowInfo, &win, &miaInfo->pResultRow, pSup, &iaInfo->aggSup); if (ret != TSDB_CODE_SUCCESS || miaInfo->pResultRow == NULL) { T_LONG_JMP(pTaskInfo->env, ret); } @@ -4963,7 +5056,7 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR currWin.ekey = taosTimeAdd(currWin.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; startPos = currPos; - ret = setSingleOutputTupleBuf(pResultRowInfo, &win, &miaInfo->pResultRow, pSup, &iaInfo->aggSup); + ret = setSingleOutputTupleBuf(pResultRowInfo, &win, &miaInfo->pResultRow, pSup, &iaInfo->aggSup); if (ret != TSDB_CODE_SUCCESS || miaInfo->pResultRow == NULL) { T_LONG_JMP(pTaskInfo->env, ret); } @@ -5032,7 +5125,7 @@ static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { pMiaInfo->prefetchedBlock = pBlock; cleanupAfterGroupResultGen(pMiaInfo, pRes); break; - } else { + } else { // continue } } @@ -5197,7 +5290,7 @@ static int32_t finalizeWindowResult(SOperatorInfo* pOperatorInfo, uint64_t table SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet( iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); ASSERT(p1 != NULL); -// finalizeResultRows(iaInfo->aggSup.pResultBuf, p1, pResultBlock, pTaskInfo); + // finalizeResultRows(iaInfo->aggSup.pResultBuf, p1, pResultBlock, pTaskInfo); tSimpleHashRemove(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); return TSDB_CODE_SUCCESS; } @@ -5222,7 +5315,7 @@ static int32_t outputPrevIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t t STimeWindow* prevWin = &prevGrpWin->window; if ((ascScan && newWin->skey > prevWin->ekey) || ((!ascScan) && newWin->skey < prevWin->ekey)) { -// finalizeWindowResult(pOperatorInfo, tableGroupId, prevWin, pResultBlock); + // finalizeWindowResult(pOperatorInfo, tableGroupId, prevWin, pResultBlock); tdListPopNode(miaInfo->groupIntervals, listNode); } } @@ -5382,7 +5475,7 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) { if (listNode != NULL) { SGroupTimeWindow* grpWin = (SGroupTimeWindow*)(listNode->data); -// finalizeWindowResult(pOperator, grpWin->groupId, &grpWin->window, pRes); + // finalizeWindowResult(pOperator, grpWin->groupId, &grpWin->window, pRes); pRes->info.groupId = grpWin->groupId; } } @@ -5591,7 +5684,7 @@ static void doStreamIntervalAggImpl2(SOperatorInfo* pOperatorInfo, SSDataBlock* forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) { - saveWinResultRow(pResult, tableGroupId, pUpdatedMap); + saveWinResultInfo(pResult->win.skey, tableGroupId, pUpdatedMap); } updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, @@ -5600,7 +5693,7 @@ static void doStreamIntervalAggImpl2(SOperatorInfo* pOperatorInfo, SSDataBlock* .ts = nextWin.skey, .groupId = tableGroupId, }; - saveOutput(pTaskInfo, &key, pResult, pInfo->aggSup.resultRowSize); + saveOutputBuf(pTaskInfo, &key, pResult, pInfo->aggSup.resultRowSize); releaseOutputBuf(pTaskInfo, &key, pResult); int32_t prevEndPos = (forwardRows - 1) * step + startPos; ASSERT(pSDataBlock->info.window.skey > 0 && pSDataBlock->info.window.ekey > 0); @@ -5645,7 +5738,8 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { return pInfo->pDelRes; } - doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); + doBuildResult(pOperator, pInfo->binfo.pRes, &pInfo->groupResInfo); + // doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainResults(&pInfo->groupResInfo)) { pOperator->status = OP_EXEC_DONE; qDebug("===stream===single interval is done"); @@ -5671,13 +5765,14 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { printDataBlock(pBlock, "single interval recv"); if (pBlock->info.type == STREAM_CLEAR) { - doClearWindows(&pInfo->aggSup, &pOperator->exprSupp, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, - NULL); + doDeleteWindows(pOperator, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, NULL, NULL); qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo)); continue; } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) { - doDeleteSpecifyIntervalWindow(&pInfo->aggSup, &pInfo->twAggSup, pBlock, pInfo->pDelWins, &pInfo->interval, - pUpdatedMap); + // doDeleteSpecifyIntervalWindow(&pInfo->aggSup, &pInfo->twAggSup, pBlock, pInfo->pDelWins, &pInfo->interval, + // pUpdatedMap); + doDeleteWindows(pOperator, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, pInfo->pDelWins, + pUpdatedMap); continue; } else if (pBlock->info.type == STREAM_GET_ALL) { getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap); @@ -5704,9 +5799,9 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { maxTs = TMAX(maxTs, pBlock->info.window.ekey); minTs = TMIN(minTs, pBlock->info.window.skey); - doStreamIntervalAggImpl(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdatedMap); + // doStreamIntervalAggImpl(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdatedMap); // new disc buf - /*doStreamIntervalAggImpl2(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap);*/ + doStreamIntervalAggImpl2(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap); } pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, minTs); @@ -5741,8 +5836,8 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { #endif pOperator->status = OP_RES_TO_RETURN; - closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pUpdatedMap, - pInfo->pRecycledPages, pInfo->aggSup.pResultBuf); + closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pUpdatedMap, + pOperator); void* pIte = NULL; while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) { @@ -5751,7 +5846,8 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { taosArraySort(pUpdated, resultrowComparAsc); // new disc buf - finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset); + // finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, + // pSup->rowEntryInfoOffset); initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); removeDeleteResults(pUpdatedMap, pInfo->pDelWins); @@ -5762,9 +5858,9 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { return pInfo->pDelRes; } - doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); + // doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); // new disc buf - // doBuildResult(pOperator, pInfo->binfo.pRes, &pInfo->groupResInfo); + doBuildResult(pOperator, pInfo->binfo.pRes, &pInfo->groupResInfo); printDataBlock(pInfo->binfo.pRes, "single interval"); return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; } @@ -5809,6 +5905,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys .calTrigger = pIntervalPhyNode->window.triggerType, .maxTs = INT64_MIN, .minTs = INT64_MAX, + .deleteMark = INT64_MAX, }; ASSERT(twAggSupp.calTrigger != STREAM_TRIGGER_MAX_DELAY); pOperator->pTaskInfo = pTaskInfo; @@ -5856,7 +5953,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys createOperatorFpSet(operatorDummyOpenFn, doStreamIntervalAgg, NULL, NULL, destroyStreamIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); - initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, pInfo->twAggSup.waterMark); + initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, &pInfo->twAggSup); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { goto _error; diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 9b502eded7..a23f58a732 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -5297,12 +5297,12 @@ bool modeFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { } static void doModeAdd(SModeInfo* pInfo, char* data) { - int32_t hashKeyBytes = IS_VAR_DATA_TYPE(pInfo->colType) ? varDataTLen(data) : pInfo->colBytes; + int32_t hashKeyBytes = IS_STR_DATA_TYPE(pInfo->colType) ? varDataTLen(data) : pInfo->colBytes; SModeItem** pHashItem = taosHashGet(pInfo->pHash, data, hashKeyBytes); if (pHashItem == NULL) { int32_t size = sizeof(SModeItem) + pInfo->colBytes; SModeItem* pItem = (SModeItem*)(pInfo->pItems + pInfo->numOfPoints * size); - memcpy(pItem->data, data, pInfo->colBytes); + memcpy(pItem->data, data, hashKeyBytes); pItem->count += 1; taosHashPut(pInfo->pHash, data, hashKeyBytes, &pItem, sizeof(SModeItem*)); diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 3f472b53a0..62427e0488 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -961,6 +961,7 @@ void releaseUdfFuncHandle(char* udfName) { strcpy(key.udfName, udfName); SUdfcFuncStub *foundStub = taosArraySearch(gUdfdProxy.udfStubs, &key, compareUdfcFuncSub, TD_EQ); if (!foundStub) { + uv_mutex_unlock(&gUdfdProxy.udfStubsMutex); return; } if (foundStub->refCount > 0) { diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index a45e4585e8..636f006d6e 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -41,6 +41,8 @@ typedef struct SUdfdContext { uv_mutex_t udfsMutex; SHashObj * udfsHash; + SArray* residentFuncs; + bool printVersion; } SUdfdContext; @@ -67,6 +69,7 @@ typedef struct SUdf { EUdfState state; uv_mutex_t lock; uv_cond_t condReady; + bool resident; char name[TSDB_FUNC_NAME_LEN]; int8_t funcType; @@ -200,6 +203,14 @@ void udfdProcessSetupRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { if (udf->initFunc) { udf->initFunc(); } + udf->resident = false; + for (int32_t i = 0; i < taosArrayGetSize(global.residentFuncs); ++i) { + char* funcName = taosArrayGet(global.residentFuncs, i); + if (strcmp(setup->udfName, funcName) == 0) { + udf->resident = true; + break; + } + } udf->state = UDF_STATE_READY; uv_cond_broadcast(&udf->condReady); uv_mutex_unlock(&udf->lock); @@ -345,7 +356,7 @@ void udfdProcessTeardownRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { uv_mutex_lock(&global.udfsMutex); udf->refCount--; - if (udf->refCount == 0) { + if (udf->refCount == 0 && !udf->resident) { unloadUdf = true; taosHashRemove(global.udfsHash, udf->name, strlen(udf->name)); } @@ -576,9 +587,9 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) { uv_dlsym(&udf->lib, finishFuncName, (void **)(&udf->aggFinishFunc)); char mergeFuncName[TSDB_FUNC_NAME_LEN + 6] = {0}; char *mergeSuffix = "_merge"; - strncpy(finishFuncName, processFuncName, sizeof(finishFuncName)); - strncat(finishFuncName, mergeSuffix, strlen(mergeSuffix)); - uv_dlsym(&udf->lib, finishFuncName, (void **)(&udf->aggMergeFunc)); + strncpy(mergeFuncName, processFuncName, sizeof(mergeFuncName)); + strncat(mergeFuncName, mergeSuffix, strlen(mergeSuffix)); + uv_dlsym(&udf->lib, mergeFuncName, (void **)(&udf->aggMergeFunc)); } return 0; } @@ -919,8 +930,6 @@ static int32_t udfdRun() { uv_run(global.loop, UV_RUN_DEFAULT); uv_loop_close(global.loop); - uv_mutex_destroy(&global.udfsMutex); - taosHashCleanup(global.udfsHash); return 0; } @@ -941,6 +950,47 @@ void udfdConnectMnodeThreadFunc(void *args) { } } +int32_t udfdInitResidentFuncs() { + if (strlen(tsUdfdResFuncs) == 0) { + return TSDB_CODE_SUCCESS; + } + + global.residentFuncs = taosArrayInit(2, TSDB_FUNC_NAME_LEN); + char* pSave = tsUdfdResFuncs; + char* token; + while ((token = strtok_r(pSave, ",", &pSave)) != NULL) { + char func[TSDB_FUNC_NAME_LEN] = {0}; + strncpy(func, token, strlen(token)); + taosArrayPush(global.residentFuncs, func); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t udfdDeinitResidentFuncs() { + for (int32_t i = 0; i < taosArrayGetSize(global.residentFuncs); ++i) { + char* funcName = taosArrayGet(global.residentFuncs, i); + SUdf** udfInHash = taosHashGet(global.udfsHash, funcName, strlen(funcName)); + if (udfInHash) { + taosHashRemove(global.udfsHash, funcName, strlen(funcName)); + SUdf* udf = *udfInHash; + if (udf->destroyFunc) { + (udf->destroyFunc)(); + } + uv_dlclose(&udf->lib); + taosMemoryFree(udf); + } + } + taosArrayDestroy(global.residentFuncs); + return TSDB_CODE_SUCCESS; +} + +int32_t udfdCleanup() { + uv_mutex_destroy(&global.udfsMutex); + taosHashCleanup(global.udfsHash); + return 0; +} + int main(int argc, char *argv[]) { if (!taosCheckSystemIsLittleEnd()) { printf("failed to start since on non-little-end machines\n"); @@ -978,6 +1028,8 @@ int main(int argc, char *argv[]) { return -5; } + udfdInitResidentFuncs(); + uv_thread_t mnodeConnectThread; uv_thread_create(&mnodeConnectThread, udfdConnectMnodeThreadFunc, NULL); @@ -986,5 +1038,7 @@ int main(int argc, char *argv[]) { removeListeningPipe(); udfdCloseClientRpc(); + udfdDeinitResidentFuncs(); + udfdCleanup(); return 0; } diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index ade5be4722..2c47ddea8b 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -17,6 +17,18 @@ #include "plannodes.h" #include "tdatablock.h" +#ifndef htonll + +#define htonll(x) \ + (((int64_t)x & 0x00000000000000ff) << 7 * 8) | (((int64_t)x & 0x000000000000ff00) << 5 * 8) | \ + (((int64_t)x & 0x0000000000ff0000) << 3 * 8) | (((int64_t)x & 0x00000000ff000000) << 1 * 8) | \ + (((int64_t)x & 0x000000ff00000000) >> 1 * 8) | (((int64_t)x & 0x0000ff0000000000) >> 3 * 8) | \ + (((int64_t)x & 0x00ff000000000000) >> 5 * 8) | (((int64_t)x & 0xff00000000000000) >> 7 * 8) + +#define ntohll(x) htonll(x) + +#endif + #define NODES_MSG_DEFAULT_LEN 1024 #define TLV_TYPE_ARRAY_ELEM 0 @@ -86,8 +98,8 @@ static int32_t tlvEncodeImpl(STlvEncoder* pEncoder, int16_t type, const void* pV pEncoder->allocSize = pEncoder->allocSize * 2; } STlv* pTlv = (STlv*)(pEncoder->pBuf + pEncoder->offset); - pTlv->type = type; - pTlv->len = len; + pTlv->type = htons(type); + pTlv->len = htonl(len); memcpy(pTlv->value, pValue, len); pEncoder->offset += tlvLen; ++(pEncoder->tlvCount); @@ -117,26 +129,32 @@ static int32_t tlvEncodeValueI8(STlvEncoder* pEncoder, int8_t value) { } static int32_t tlvEncodeI16(STlvEncoder* pEncoder, int16_t type, int16_t value) { + value = htons(value); return tlvEncodeImpl(pEncoder, type, &value, sizeof(value)); } static int32_t tlvEncodeValueI16(STlvEncoder* pEncoder, int16_t value) { + value = htons(value); return tlvEncodeValueImpl(pEncoder, &value, sizeof(value)); } static int32_t tlvEncodeI32(STlvEncoder* pEncoder, int16_t type, int32_t value) { + value = htonl(value); return tlvEncodeImpl(pEncoder, type, &value, sizeof(value)); } static int32_t tlvEncodeValueI32(STlvEncoder* pEncoder, int32_t value) { + value = htonl(value); return tlvEncodeValueImpl(pEncoder, &value, sizeof(value)); } static int32_t tlvEncodeI64(STlvEncoder* pEncoder, int16_t type, int64_t value) { + value = htonll(value); return tlvEncodeImpl(pEncoder, type, &value, sizeof(value)); } static int32_t tlvEncodeValueI64(STlvEncoder* pEncoder, int64_t value) { + value = htonll(value); return tlvEncodeValueImpl(pEncoder, &value, sizeof(value)); } @@ -149,34 +167,44 @@ static int32_t tlvEncodeValueU8(STlvEncoder* pEncoder, uint8_t value) { } static int32_t tlvEncodeU16(STlvEncoder* pEncoder, int16_t type, uint16_t value) { + value = htons(value); return tlvEncodeImpl(pEncoder, type, &value, sizeof(value)); } static int32_t tlvEncodeValueU16(STlvEncoder* pEncoder, uint16_t value) { + value = htons(value); return tlvEncodeValueImpl(pEncoder, &value, sizeof(value)); } static int32_t tlvEncodeU64(STlvEncoder* pEncoder, int16_t type, uint64_t value) { + value = htonll(value); return tlvEncodeImpl(pEncoder, type, &value, sizeof(value)); } static int32_t tlvEncodeValueU64(STlvEncoder* pEncoder, uint64_t value) { + value = htonll(value); return tlvEncodeValueImpl(pEncoder, &value, sizeof(value)); } static int32_t tlvEncodeDouble(STlvEncoder* pEncoder, int16_t type, double value) { - return tlvEncodeImpl(pEncoder, type, &value, sizeof(value)); + int64_t temp = *(int64_t*)&value; + temp = htonll(temp); + return tlvEncodeImpl(pEncoder, type, &temp, sizeof(temp)); } static int32_t tlvEncodeValueDouble(STlvEncoder* pEncoder, double value) { - return tlvEncodeValueImpl(pEncoder, &value, sizeof(value)); + int64_t temp = *(int64_t*)&value; + temp = htonll(temp); + return tlvEncodeValueImpl(pEncoder, &temp, sizeof(temp)); } static int32_t tlvEncodeEnum(STlvEncoder* pEncoder, int16_t type, int32_t value) { + value = htonl(value); return tlvEncodeImpl(pEncoder, type, &value, sizeof(value)); } static int32_t tlvEncodeValueEnum(STlvEncoder* pEncoder, int32_t value) { + value = htonl(value); return tlvEncodeValueImpl(pEncoder, &value, sizeof(value)); } @@ -197,7 +225,7 @@ static int32_t tlvEncodeCStr(STlvEncoder* pEncoder, int16_t type, const char* pV static int32_t tlvEncodeValueCStr(STlvEncoder* pEncoder, const char* pValue) { int16_t len = strlen(pValue); - int32_t code = tlvEncodeValueImpl(pEncoder, &len, sizeof(len)); + int32_t code = tlvEncodeValueI16(pEncoder, len); if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeValueImpl(pEncoder, pValue, len); } @@ -218,8 +246,8 @@ static int32_t tlvEncodeObj(STlvEncoder* pEncoder, int16_t type, FToMsg func, co int32_t code = func(pObj, pEncoder); if (TSDB_CODE_SUCCESS == code) { STlv* pTlv = (STlv*)(pEncoder->pBuf + start); - pTlv->type = type; - pTlv->len = pEncoder->offset - start - sizeof(STlv); + pTlv->type = htons(type); + pTlv->len = htonl(pEncoder->offset - start - sizeof(STlv)); } ++(pEncoder->tlvCount); return code; @@ -236,8 +264,8 @@ static int32_t tlvEncodeObjArray(STlvEncoder* pEncoder, int16_t type, FToMsg fun } if (TSDB_CODE_SUCCESS == code) { STlv* pTlv = (STlv*)(pEncoder->pBuf + start); - pTlv->type = type; - pTlv->len = pEncoder->offset - start - sizeof(STlv); + pTlv->type = htons(type); + pTlv->len = htonl(pEncoder->offset - start - sizeof(STlv)); } } return code; @@ -259,6 +287,8 @@ static int32_t tlvGetNextTlv(STlvDecoder* pDecoder, STlv** pTlv) { } *pTlv = (STlv*)(pDecoder->pBuf + pDecoder->offset); + (*pTlv)->type = ntohs((*pTlv)->type); + (*pTlv)->len = ntohl((*pTlv)->len); if ((*pTlv)->len + pDecoder->offset > pDecoder->bufSize) { return TSDB_CODE_FAILED; } @@ -291,22 +321,52 @@ static int32_t tlvDecodeValueI8(STlvDecoder* pDecoder, int8_t* pValue) { return tlvDecodeValueImpl(pDecoder, pValue, sizeof(*pValue)); } -static int32_t tlvDecodeI16(STlv* pTlv, int16_t* pValue) { return tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); } +static int32_t tlvDecodeI16(STlv* pTlv, int16_t* pValue) { + int32_t code = tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); + if (TSDB_CODE_SUCCESS == code) { + *pValue = ntohs(*pValue); + } + return code; +} static int32_t tlvDecodeValueI16(STlvDecoder* pDecoder, int16_t* pValue) { - return tlvDecodeValueImpl(pDecoder, pValue, sizeof(*pValue)); + int32_t code = tlvDecodeValueImpl(pDecoder, pValue, sizeof(*pValue)); + if (TSDB_CODE_SUCCESS == code) { + *pValue = ntohs(*pValue); + } + return code; } -static int32_t tlvDecodeI32(STlv* pTlv, int32_t* pValue) { return tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); } +static int32_t tlvDecodeI32(STlv* pTlv, int32_t* pValue) { + int32_t code = tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); + if (TSDB_CODE_SUCCESS == code) { + *pValue = ntohl(*pValue); + } + return code; +} static int32_t tlvDecodeValueI32(STlvDecoder* pDecoder, int32_t* pValue) { - return tlvDecodeValueImpl(pDecoder, pValue, sizeof(*pValue)); + int32_t code = tlvDecodeValueImpl(pDecoder, pValue, sizeof(*pValue)); + if (TSDB_CODE_SUCCESS == code) { + *pValue = ntohl(*pValue); + } + return code; } -static int32_t tlvDecodeI64(STlv* pTlv, int64_t* pValue) { return tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); } +static int32_t tlvDecodeI64(STlv* pTlv, int64_t* pValue) { + int32_t code = tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); + if (TSDB_CODE_SUCCESS == code) { + *pValue = ntohll(*pValue); + } + return code; +} static int32_t tlvDecodeValueI64(STlvDecoder* pDecoder, int64_t* pValue) { - return tlvDecodeValueImpl(pDecoder, pValue, sizeof(*pValue)); + int32_t code = tlvDecodeValueImpl(pDecoder, pValue, sizeof(*pValue)); + if (TSDB_CODE_SUCCESS == code) { + *pValue = ntohll(*pValue); + } + return code; } static int32_t tlvDecodeU8(STlv* pTlv, uint8_t* pValue) { return tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); } @@ -315,22 +375,54 @@ static int32_t tlvDecodeValueU8(STlvDecoder* pDecoder, uint8_t* pValue) { return tlvDecodeValueImpl(pDecoder, pValue, sizeof(*pValue)); } -static int32_t tlvDecodeU16(STlv* pTlv, uint16_t* pValue) { return tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); } +static int32_t tlvDecodeU16(STlv* pTlv, uint16_t* pValue) { + int32_t code = tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); + if (TSDB_CODE_SUCCESS == code) { + *pValue = ntohs(*pValue); + } + return code; +} static int32_t tlvDecodeValueU16(STlvDecoder* pDecoder, uint16_t* pValue) { - return tlvDecodeValueImpl(pDecoder, pValue, sizeof(*pValue)); + int32_t code = tlvDecodeValueImpl(pDecoder, pValue, sizeof(*pValue)); + if (TSDB_CODE_SUCCESS == code) { + *pValue = ntohs(*pValue); + } + return code; } -static int32_t tlvDecodeU64(STlv* pTlv, uint64_t* pValue) { return tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); } +static int32_t tlvDecodeU64(STlv* pTlv, uint64_t* pValue) { + int32_t code = tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); + if (TSDB_CODE_SUCCESS == code) { + *pValue = ntohll(*pValue); + } + return code; +} static int32_t tlvDecodeValueU64(STlvDecoder* pDecoder, uint64_t* pValue) { - return tlvDecodeValueImpl(pDecoder, pValue, sizeof(*pValue)); + int32_t code = tlvDecodeValueImpl(pDecoder, pValue, sizeof(*pValue)); + if (TSDB_CODE_SUCCESS == code) { + *pValue = ntohll(*pValue); + } + return code; } -static int32_t tlvDecodeDouble(STlv* pTlv, double* pValue) { return tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); } +static int32_t tlvDecodeDouble(STlv* pTlv, double* pValue) { + int64_t temp = 0; + int32_t code = tlvDecodeI64(pTlv, &temp); + if (TSDB_CODE_SUCCESS == code) { + *pValue = *(double*)&temp; + } + return code; +} static int32_t tlvDecodeValueDouble(STlvDecoder* pDecoder, double* pValue) { - return tlvDecodeValueImpl(pDecoder, pValue, sizeof(*pValue)); + int64_t temp = 0; + int32_t code = tlvDecodeValueI64(pDecoder, &temp); + if (TSDB_CODE_SUCCESS == code) { + *pValue = *(double*)&temp; + } + return code; } static int32_t convertIntegerType(int32_t value, void* pValue, int16_t len) { @@ -2462,33 +2554,54 @@ static int32_t msgToPhysiWindowNode(STlvDecoder* pDecoder, void* pObj) { return code; } -enum { - PHY_INTERVAL_CODE_WINDOW = 1, - PHY_INTERVAL_CODE_INTERVAL, - PHY_INTERVAL_CODE_OFFSET, - PHY_INTERVAL_CODE_SLIDING, - PHY_INTERVAL_CODE_INTERVAL_UNIT, - PHY_INTERVAL_CODE_SLIDING_UNIT -}; +enum { PHY_INTERVAL_CODE_WINDOW = 1, PHY_INTERVAL_CODE_INLINE_ATTRS }; + +static int32_t physiIntervalNodeInlineToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SIntervalPhysiNode* pNode = (const SIntervalPhysiNode*)pObj; + + int32_t code = tlvEncodeValueI64(pEncoder, pNode->interval); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeValueI64(pEncoder, pNode->offset); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeValueI64(pEncoder, pNode->sliding); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeValueI8(pEncoder, pNode->intervalUnit); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeValueI8(pEncoder, pNode->slidingUnit); + } + + return code; +} static int32_t physiIntervalNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { const SIntervalPhysiNode* pNode = (const SIntervalPhysiNode*)pObj; int32_t code = tlvEncodeObj(pEncoder, PHY_INTERVAL_CODE_WINDOW, physiWindowNodeToMsg, &pNode->window); if (TSDB_CODE_SUCCESS == code) { - code = tlvEncodeI64(pEncoder, PHY_INTERVAL_CODE_INTERVAL, pNode->interval); + code = tlvEncodeObj(pEncoder, PHY_INTERVAL_CODE_INLINE_ATTRS, physiIntervalNodeInlineToMsg, pNode); + } + + return code; +} + +static int32_t msgToPhysiIntervalNodeInline(STlvDecoder* pDecoder, void* pObj) { + SIntervalPhysiNode* pNode = (SIntervalPhysiNode*)pObj; + + int32_t code = tlvDecodeValueI64(pDecoder, &pNode->interval); + if (TSDB_CODE_SUCCESS == code) { + code = tlvDecodeValueI64(pDecoder, &pNode->offset); } if (TSDB_CODE_SUCCESS == code) { - code = tlvEncodeI64(pEncoder, PHY_INTERVAL_CODE_OFFSET, pNode->offset); + code = tlvDecodeValueI64(pDecoder, &pNode->sliding); } if (TSDB_CODE_SUCCESS == code) { - code = tlvEncodeI64(pEncoder, PHY_INTERVAL_CODE_SLIDING, pNode->sliding); + code = tlvDecodeValueI8(pDecoder, &pNode->intervalUnit); } if (TSDB_CODE_SUCCESS == code) { - code = tlvEncodeI8(pEncoder, PHY_INTERVAL_CODE_INTERVAL_UNIT, pNode->intervalUnit); - } - if (TSDB_CODE_SUCCESS == code) { - code = tlvEncodeI8(pEncoder, PHY_INTERVAL_CODE_SLIDING_UNIT, pNode->slidingUnit); + code = tlvDecodeValueI8(pDecoder, &pNode->slidingUnit); } return code; @@ -2504,20 +2617,8 @@ static int32_t msgToPhysiIntervalNode(STlvDecoder* pDecoder, void* pObj) { case PHY_INTERVAL_CODE_WINDOW: code = tlvDecodeObjFromTlv(pTlv, msgToPhysiWindowNode, &pNode->window); break; - case PHY_INTERVAL_CODE_INTERVAL: - code = tlvDecodeI64(pTlv, &pNode->interval); - break; - case PHY_INTERVAL_CODE_OFFSET: - code = tlvDecodeI64(pTlv, &pNode->offset); - break; - case PHY_INTERVAL_CODE_SLIDING: - code = tlvDecodeI64(pTlv, &pNode->sliding); - break; - case PHY_INTERVAL_CODE_INTERVAL_UNIT: - code = tlvDecodeI8(pTlv, &pNode->intervalUnit); - break; - case PHY_INTERVAL_CODE_SLIDING_UNIT: - code = tlvDecodeI8(pTlv, &pNode->slidingUnit); + case PHY_INTERVAL_CODE_INLINE_ATTRS: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiIntervalNodeInline, pNode); break; default: break; diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 1af61b0ca8..ca9bb16c05 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1059,7 +1059,7 @@ end: for (int i = 0; i < taosArrayGetSize(pTagVals); ++i) { STagVal* p = (STagVal*)taosArrayGet(pTagVals, i); if (IS_VAR_DATA_TYPE(p->type)) { - taosMemoryFree(p->pData); + taosMemoryFreeClear(p->pData); } } taosArrayDestroy(pTagVals); @@ -2040,7 +2040,7 @@ end: for (int i = 0; i < taosArrayGetSize(pTagArray); ++i) { STagVal* p = (STagVal*)taosArrayGet(pTagArray, i); if (p->type == TSDB_DATA_TYPE_NCHAR) { - taosMemoryFree(p->pData); + taosMemoryFreeClear(p->pData); } } taosArrayDestroy(pTagArray); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index ea9eefbb29..f0d5e5d67e 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -6664,12 +6664,7 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS break; } } while (0); - for (int i = 0; i < taosArrayGetSize(pTagVals); ++i) { - STagVal* p = (STagVal*)taosArrayGet(pTagVals, i); - if (IS_VAR_DATA_TYPE(p->type)) { - taosMemoryFree(p->pData); - } - } + taosArrayDestroy(pTagVals); if (code != TSDB_CODE_SUCCESS) { return code; diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index daab80667c..3075be1a00 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -410,6 +410,12 @@ end: if (retCode == TSDB_CODE_SUCCESS) { tTagNew(pTagVals, 1, true, ppTag); } + for (int i = 0; i < taosArrayGetSize(pTagVals); ++i) { + STagVal* p = (STagVal*)taosArrayGet(pTagVals, i); + if (IS_VAR_DATA_TYPE(p->type)) { + taosMemoryFreeClear(p->pData); + } + } cJSON_Delete(root); return retCode; } diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index beb938b161..a3cb0c2654 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -915,20 +915,30 @@ static int32_t stbSplSplitSortNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) } static int32_t stbSplSplitScanNodeWithoutPartTags(SSplitContext* pCxt, SStableSplitInfo* pInfo) { - int32_t code = splCreateExchangeNodeForSubplan(pCxt, pInfo->pSubplan, pInfo->pSplitNode, SUBPLAN_TYPE_MERGE); + SLogicNode* pSplitNode = pInfo->pSplitNode; + if (NULL != pInfo->pSplitNode->pParent && QUERY_NODE_LOGIC_PLAN_PROJECT == nodeType(pInfo->pSplitNode->pParent) && + NULL == pInfo->pSplitNode->pParent->pLimit && NULL == pInfo->pSplitNode->pParent->pSlimit) { + pSplitNode = pInfo->pSplitNode->pParent; + } + int32_t code = splCreateExchangeNodeForSubplan(pCxt, pInfo->pSubplan, pSplitNode, SUBPLAN_TYPE_MERGE); if (TSDB_CODE_SUCCESS == code) { code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren, - (SNode*)splCreateScanSubplan(pCxt, pInfo->pSplitNode, SPLIT_FLAG_STABLE_SPLIT)); + (SNode*)splCreateScanSubplan(pCxt, pSplitNode, SPLIT_FLAG_STABLE_SPLIT)); } ++(pCxt->groupId); return code; } static int32_t stbSplSplitScanNodeWithPartTags(SSplitContext* pCxt, SStableSplitInfo* pInfo) { - int32_t code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pInfo->pSplitNode, NULL, pInfo->pSplitNode, true); + SLogicNode* pSplitNode = pInfo->pSplitNode; + if (NULL != pInfo->pSplitNode->pParent && QUERY_NODE_LOGIC_PLAN_PROJECT == nodeType(pInfo->pSplitNode->pParent) && + NULL == pInfo->pSplitNode->pParent->pLimit && NULL == pInfo->pSplitNode->pParent->pSlimit) { + pSplitNode = pInfo->pSplitNode->pParent; + } + int32_t code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pSplitNode, NULL, pSplitNode, true); if (TSDB_CODE_SUCCESS == code) { code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren, - (SNode*)splCreateScanSubplan(pCxt, pInfo->pSplitNode, SPLIT_FLAG_STABLE_SPLIT)); + (SNode*)splCreateScanSubplan(pCxt, pSplitNode, SPLIT_FLAG_STABLE_SPLIT)); } pInfo->pSubplan->subplanType = SUBPLAN_TYPE_MERGE; ++(pCxt->groupId); diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index bf19c7a222..2b8e3d9864 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -473,10 +473,11 @@ class PlannerTestBaseImpl { cout << "nodesNodeToMsg: " << chrono::duration_cast(chrono::steady_clock::now() - start).count() << "us" << endl; + string copyStr(pStr, len); SNode* pNode = NULL; char* pNewStr = NULL; int32_t newlen = 0; - DO_WITH_THROW(nodesMsgToNode, pStr, len, &pNode) + DO_WITH_THROW(nodesMsgToNode, copyStr.c_str(), len, &pNode) DO_WITH_THROW(nodesNodeToMsg, pNode, &pNewStr, &newlen) if (newlen != len || 0 != memcmp(pStr, pNewStr, len)) { cout << "nodesNodeToMsg error!!!!!!!!!!!!!! len = " << len << ", newlen = " << newlen << endl; diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index ecc25e9c2d..c38c9ac1cf 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -632,6 +632,7 @@ int32_t qWorkerProcessDeleteMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, SD QW_ERR_JRET(qwProcessDelete(QW_FPARAMS(), &qwMsg, pRes)); + taosMemoryFreeClear(req.msg); QW_SCH_TASK_DLOG("processDelete end, node:%p", node); _return: diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 6cd5132bb9..fde8bca77b 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -18,14 +18,19 @@ #include "tcommon.h" #include "ttimer.h" -SStreamState* streamStateOpen(char* path, SStreamTask* pTask) { +SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath) { SStreamState* pState = taosMemoryCalloc(1, sizeof(SStreamState)); if (pState == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } + char statePath[300]; - sprintf(statePath, "%s/%d", path, pTask->taskId); + if (!specPath) { + sprintf(statePath, "%s/%d", path, pTask->taskId); + } else { + memcpy(statePath, path, 300); + } if (tdbOpen(statePath, 4096, 256, &pState->db) < 0) { goto _err; } @@ -135,15 +140,9 @@ int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void* if (streamStateGet(pState, key, pVal, pVLen) == 0) { return 0; } - void* tmp = taosMemoryCalloc(1, size); - if (streamStatePut(pState, key, &tmp, size) == 0) { - taosMemoryFree(tmp); - int32_t code = streamStateGet(pState, key, pVal, pVLen); - ASSERT(code == 0); - return code; - } - taosMemoryFree(tmp); - return -1; + *pVal = tdbRealloc(NULL, size); + memset(*pVal, 0, size); + return 0; } int32_t streamStateReleaseBuf(SStreamState* pState, const SWinKey* key, void* pVal) { @@ -191,9 +190,14 @@ SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key if (pCur == NULL) { return NULL; } + if (tdbTbcOpen(pState->pStateDb, &pCur->pCur, NULL) < 0) { + taosMemoryFree(pCur); + return NULL; + } int32_t c; if (tdbTbcMoveTo(pCur->pCur, key, sizeof(SWinKey), &c) < 0) { + tdbTbcClose(pCur->pCur); taosMemoryFree(pCur); return NULL; } @@ -212,9 +216,14 @@ SStreamStateCur* streamStateSeekKeyPrev(SStreamState* pState, const SWinKey* key if (pCur == NULL) { return NULL; } + if (tdbTbcOpen(pState->pStateDb, &pCur->pCur, NULL) < 0) { + taosMemoryFree(pCur); + return NULL; + } int32_t c; if (tdbTbcMoveTo(pCur->pCur, key, sizeof(SWinKey), &c) < 0) { + tdbTbcClose(pCur->pCur); taosMemoryFree(pCur); return NULL; } diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index c6ecd37680..c5204ef59e 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -841,6 +841,10 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx // copy content to the parent page tdbBtreeInitPage(pParent, &(SBtreeInitPageArg){.flags = flags, .pBt = pBt}, 0); tdbPageCopy(pNews[0], pParent, 1); + + if (!TDB_BTREE_PAGE_IS_LEAF(pNews[0])) { + ((SIntHdr *)(pParent->pData))->pgno = ((SIntHdr *)(pNews[0]->pData))->pgno; + } } for (int i = 0; i < 3; i++) { diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 2cc62d3d6a..04711eb6a0 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -248,6 +248,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { return 0; } + // loop to write the dirty pages to file SRBTreeIter iter = tRBTreeIterCreate(&pPager->rbt, 1); SRBTreeNode *pNode = NULL; while ((pNode = tRBTreeIterNext(&iter)) != NULL) { @@ -257,37 +258,23 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { ASSERT(0); return -1; } + } + + tdbTrace("tdbttl commit:%p, %d/%d", pPager, pPager->dbOrigSize, pPager->dbFileSize); + pPager->dbOrigSize = pPager->dbFileSize; + + // release the page + iter = tRBTreeIterCreate(&pPager->rbt, 1); + while ((pNode = tRBTreeIterNext(&iter)) != NULL) { + pPage = (SPage *)pNode; pPage->isDirty = 0; - // tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage); + tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage); tdbPCacheRelease(pPager->pCache, pPage, pTxn); } tRBTreeCreate(&pPager->rbt, pageCmpFn); - /* - // loop to write the dirty pages to file - for (pPage = pPager->pDirty; pPage; pPage = pPage->pDirtyNext) { - // TODO: update the page footer - ret = tdbPagerWritePageToDB(pPager, pPage); - if (ret < 0) { - ASSERT(0); - return -1; - } - } - - // release the page - for (pPage = pPager->pDirty; pPage; pPage = pPager->pDirty) { - pPager->pDirty = pPage->pDirtyNext; - pPage->pDirtyNext = NULL; - - pPage->isDirty = 0; - - tdbPCacheRelease(pPager->pCache, pPage, pTxn); - } - */ - tdbTrace("tdbttl commit:%p, %d", pPager, pPager->dbOrigSize); - pPager->dbOrigSize = pPager->dbFileSize; // sync the db file tdbOsFSync(pPager->fd); @@ -353,7 +340,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { pPage->isDirty = 0; - // tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage); + tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage); tdbPCacheRelease(pPager->pCache, pPage, pTxn); } diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 93ced912f8..5284aeff77 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -268,7 +268,7 @@ int walRollFileInfo(SWal* pWal) { char* walMetaSerialize(SWal* pWal) { char buf[30]; ASSERT(pWal->fileInfoSet); - int sz = pWal->fileInfoSet->size; + int sz = taosArrayGetSize(pWal->fileInfoSet); cJSON* pRoot = cJSON_CreateObject(); cJSON* pMeta = cJSON_CreateObject(); cJSON* pFiles = cJSON_CreateArray(); @@ -384,8 +384,10 @@ static int walFindCurMetaVer(SWal* pWal) { int code = regexec(&walMetaRegexPattern, name, 0, NULL, 0); if (code == 0) { sscanf(name, "meta-ver%d", &metaVer); + wDebug("vgId:%d, wal find current meta: %s is the meta file, ver %d", pWal->cfg.vgId, name, metaVer); break; } + wDebug("vgId:%d, wal find current meta: %s is not meta file", pWal->cfg.vgId, name); } taosCloseDir(&pDir); regfree(&walMetaRegexPattern); @@ -422,6 +424,7 @@ int walLoadMeta(SWal* pWal) { // find existing meta file int metaVer = walFindCurMetaVer(pWal); if (metaVer == -1) { + wDebug("vgId:%d wal find meta ver %d", pWal->cfg.vgId, metaVer); return -1; } char fnameStr[WAL_FILE_LEN]; diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index ba877915b1..62b9d87628 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -50,6 +50,7 @@ #define _DEFAULT_SOURCE #include "tcompression.h" #include "lz4.h" +#include "tRealloc.h" #include "tlog.h" #ifdef TD_TSZ @@ -814,24 +815,24 @@ int32_t tsCompressFloatImp(const char *const input, const int32_t nelements, cha uint32_t predicted = prev_value; uint32_t diff = curr.bits ^ predicted; - int32_t leading_zeros = FLOAT_BYTES * BITS_PER_BYTE; - int32_t trailing_zeros = leading_zeros; + int32_t clz = FLOAT_BYTES * BITS_PER_BYTE; + int32_t ctz = clz; if (diff) { - trailing_zeros = BUILDIN_CTZ(diff); - leading_zeros = BUILDIN_CLZ(diff); + ctz = BUILDIN_CTZ(diff); + clz = BUILDIN_CLZ(diff); } uint8_t nbytes = 0; uint8_t flag; - if (trailing_zeros > leading_zeros) { - nbytes = (uint8_t)(FLOAT_BYTES - trailing_zeros / BITS_PER_BYTE); + if (ctz > clz) { + nbytes = (uint8_t)(FLOAT_BYTES - ctz / BITS_PER_BYTE); if (nbytes > 0) nbytes--; flag = ((uint8_t)1 << 3) | nbytes; } else { - nbytes = (uint8_t)(FLOAT_BYTES - leading_zeros / BITS_PER_BYTE); + nbytes = (uint8_t)(FLOAT_BYTES - clz / BITS_PER_BYTE); if (nbytes > 0) nbytes--; flag = nbytes; } @@ -994,3 +995,621 @@ int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, co return tdszDecompress(SZ_DOUBLE, input + 1, compressedSize - 1, nelements, output); } #endif + +/************************************************************************* + * STREAM COMPRESSION + *************************************************************************/ +#define I64_SAFE_ADD(a, b) (((a) >= 0 && (b) <= INT64_MAX - (b)) || ((a) < 0 && (b) >= INT64_MIN - (a))) +typedef struct SCompressor SCompressor; + +static int32_t tCompBool(SCompressor *pCmprsor, const void *pData, int32_t nData); +static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData); +static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nData); +static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nData); +static int32_t tCompTimestamp(SCompressor *pCmprsor, const void *pData, int32_t nData); +static int32_t tCompBinary(SCompressor *pCmprsor, const void *pData, int32_t nData); +static struct { + int8_t type; + int32_t bytes; + int8_t isVarLen; + int32_t (*cmprFn)(SCompressor *, const void *, int32_t nData); +} DATA_TYPE_INFO[] = { + {TSDB_DATA_TYPE_NULL, 0, 0, NULL}, // TSDB_DATA_TYPE_NULL + {TSDB_DATA_TYPE_BOOL, 1, 0, tCompBool}, // TSDB_DATA_TYPE_BOOL + {TSDB_DATA_TYPE_TINYINT, 1, 0, tCompInt}, // TSDB_DATA_TYPE_TINYINT + {TSDB_DATA_TYPE_SMALLINT, 2, 0, tCompInt}, // TSDB_DATA_TYPE_SMALLINT + {TSDB_DATA_TYPE_INT, 4, 0, tCompInt}, // TSDB_DATA_TYPE_INT + {TSDB_DATA_TYPE_BIGINT, 8, 0, tCompInt}, // TSDB_DATA_TYPE_BIGINT + {TSDB_DATA_TYPE_FLOAT, 4, 0, tCompFloat}, // TSDB_DATA_TYPE_FLOAT + {TSDB_DATA_TYPE_DOUBLE, 8, 0, tCompDouble}, // TSDB_DATA_TYPE_DOUBLE + {TSDB_DATA_TYPE_VARCHAR, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_VARCHAR + {TSDB_DATA_TYPE_TIMESTAMP, 8, 0, tCompTimestamp}, // pTSDB_DATA_TYPE_TIMESTAMP + {TSDB_DATA_TYPE_NCHAR, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_NCHAR + {TSDB_DATA_TYPE_UTINYINT, 1, 0, tCompInt}, // TSDB_DATA_TYPE_UTINYINT + {TSDB_DATA_TYPE_USMALLINT, 2, 0, tCompInt}, // TSDB_DATA_TYPE_USMALLINT + {TSDB_DATA_TYPE_UINT, 4, 0, tCompInt}, // TSDB_DATA_TYPE_UINT + {TSDB_DATA_TYPE_UBIGINT, 8, 0, tCompInt}, // TSDB_DATA_TYPE_UBIGINT + {TSDB_DATA_TYPE_JSON, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_JSON + {TSDB_DATA_TYPE_VARBINARY, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_VARBINARY + {TSDB_DATA_TYPE_DECIMAL, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_DECIMAL + {TSDB_DATA_TYPE_BLOB, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_BLOB + {TSDB_DATA_TYPE_MEDIUMBLOB, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_MEDIUMBLOB +}; + +struct SCompressor { + int8_t type; + int8_t cmprAlg; + int8_t autoAlloc; + int32_t nVal; + uint8_t *aBuf[2]; + int64_t nBuf[2]; + union { + // Timestamp ---- + struct { + int64_t ts_prev_val; + int64_t ts_prev_delta; + uint8_t *ts_flag_p; + }; + // Integer ---- + struct { + int64_t i_prev; + int32_t i_selector; + int32_t i_start; + int32_t i_end; + uint64_t i_aZigzag[241]; + int8_t i_aBitN[241]; + }; + // Float ---- + struct { + uint32_t f_prev; + uint8_t *f_flag_p; + }; + // Double ---- + struct { + uint64_t d_prev; + uint8_t *d_flag_p; + }; + }; +}; + +// Timestamp ===================================================== +static int32_t tCompSetCopyMode(SCompressor *pCmprsor) { + int32_t code = 0; + + if (pCmprsor->nVal) { + if (pCmprsor->autoAlloc) { + code = tRealloc(&pCmprsor->aBuf[1], sizeof(int64_t) * pCmprsor->nVal); + if (code) return code; + } + pCmprsor->nBuf[1] = 0; + + int64_t n = 1; + int64_t value; + int64_t delta; + uint64_t vZigzag; + while (n < pCmprsor->nBuf[0]) { + uint8_t aN[2]; + aN[0] = pCmprsor->aBuf[0][n] & 0xf; + aN[1] = pCmprsor->aBuf[0][n] >> 4; + + n++; + + for (int32_t i = 0; i < 2; i++) { + vZigzag = 0; + for (uint8_t j = 0; j < aN[i]; j++) { + vZigzag |= (((uint64_t)pCmprsor->aBuf[0][n]) << (8 * j)); + n++; + } + + int64_t delta_of_delta = ZIGZAG_DECODE(int64_t, vZigzag); + if (pCmprsor->nBuf[1] == 0) { + delta = 0; + value = delta_of_delta; + } else { + delta = delta_of_delta + delta; + value = delta + value; + } + + memcpy(pCmprsor->aBuf[1] + pCmprsor->nBuf[1], &value, sizeof(int64_t)); + pCmprsor->nBuf[1] += sizeof(int64_t); + + if (n >= pCmprsor->nBuf[0]) break; + } + } + + ASSERT(n == pCmprsor->nBuf[0]); + + if (pCmprsor->autoAlloc) { + code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[1] + 1); + if (code) return code; + } + memcpy(pCmprsor->aBuf[0] + 1, pCmprsor->aBuf[1], pCmprsor->nBuf[1]); + pCmprsor->nBuf[0] = 1 + pCmprsor->nBuf[1]; + } + pCmprsor->aBuf[0][0] = 0; + + return code; +} +static int32_t tCompTimestamp(SCompressor *pCmprsor, const void *pData, int32_t nData) { + int32_t code = 0; + + int64_t ts = *(int64_t *)pData; + ASSERT(pCmprsor->type == TSDB_DATA_TYPE_TIMESTAMP); + ASSERT(nData == 8); + + if (pCmprsor->aBuf[0][0] == 1) { + if (pCmprsor->nVal == 0) { + pCmprsor->ts_prev_val = ts; + pCmprsor->ts_prev_delta = -ts; + } + + if (!I64_SAFE_ADD(ts, -pCmprsor->ts_prev_val)) { + code = tCompSetCopyMode(pCmprsor); + if (code) return code; + goto _copy_cmpr; + } + int64_t delta = ts - pCmprsor->ts_prev_val; + + if (!I64_SAFE_ADD(delta, -pCmprsor->ts_prev_delta)) { + code = tCompSetCopyMode(pCmprsor); + if (code) return code; + goto _copy_cmpr; + } + int64_t delta_of_delta = delta - pCmprsor->ts_prev_delta; + uint64_t vZigzag = ZIGZAG_ENCODE(int64_t, delta_of_delta); + + pCmprsor->ts_prev_val = ts; + pCmprsor->ts_prev_delta = delta; + + if ((pCmprsor->nVal & 0x1) == 0) { + if (pCmprsor->autoAlloc) { + code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 17); + if (code) return code; + } + + pCmprsor->ts_flag_p = pCmprsor->aBuf[0] + pCmprsor->nBuf[0]; + pCmprsor->nBuf[0]++; + pCmprsor->ts_flag_p[0] = 0; + while (vZigzag) { + pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (vZigzag & 0xff); + pCmprsor->nBuf[0]++; + pCmprsor->ts_flag_p[0]++; + vZigzag >>= 8; + } + } else { + while (vZigzag) { + pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (vZigzag & 0xff); + pCmprsor->nBuf[0]++; + pCmprsor->ts_flag_p[0] += 0x10; + vZigzag >>= 8; + } + } + } else { + _copy_cmpr: + if (pCmprsor->autoAlloc) { + code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + sizeof(ts)); + if (code) return code; + } + + memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], &ts, sizeof(ts)); + pCmprsor->nBuf[0] += sizeof(ts); + } + pCmprsor->nVal++; + + return code; +} + +// Integer ===================================================== +#define SIMPLE8B_MAX ((uint64_t)1152921504606846974LL) +static const uint8_t BIT_PER_INTEGER[] = {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 15, 20, 30, 60}; +static const int32_t SELECTOR_TO_ELEMS[] = {240, 120, 60, 30, 20, 15, 12, 10, 8, 7, 6, 5, 4, 3, 2, 1}; +static const uint8_t BIT_TO_SELECTOR[] = {0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 11, 11, 12, 12, 12, + 13, 13, 13, 13, 13, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 15, + 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, + 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15}; + +static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) { + int32_t code = 0; + + ASSERT(nData == DATA_TYPE_INFO[pCmprsor->type].bytes); + + if (pCmprsor->aBuf[0][0] == 0) { + int64_t val; + + switch (pCmprsor->type) { + case TSDB_DATA_TYPE_TINYINT: + val = *(int8_t *)pData; + break; + case TSDB_DATA_TYPE_SMALLINT: + val = *(int16_t *)pData; + break; + case TSDB_DATA_TYPE_INT: + val = *(int32_t *)pData; + break; + case TSDB_DATA_TYPE_BIGINT: + val = *(int64_t *)pData; + break; + case TSDB_DATA_TYPE_UTINYINT: + val = *(uint8_t *)pData; + break; + case TSDB_DATA_TYPE_USMALLINT: + val = *(uint16_t *)pData; + break; + case TSDB_DATA_TYPE_UINT: + val = *(uint32_t *)pData; + break; + case TSDB_DATA_TYPE_UBIGINT: + val = *(int64_t *)pData; + break; + default: + ASSERT(0); + break; + } + + if (!I64_SAFE_ADD(val, -pCmprsor->i_prev)) { + // TODO + goto _copy_cmpr; + } + + int64_t diff = val - pCmprsor->i_prev; + uint64_t vZigzag = ZIGZAG_ENCODE(int64_t, diff); + if (vZigzag >= SIMPLE8B_MAX) { + // TODO + goto _copy_cmpr; + } + + int8_t nBit = (vZigzag) ? (64 - BUILDIN_CLZL(vZigzag)) : 0; + pCmprsor->i_prev = val; + + while (1) { + int32_t nEle = (pCmprsor->i_end + 241 - pCmprsor->i_start) % 241; + + if (nEle + 1 <= SELECTOR_TO_ELEMS[pCmprsor->i_selector] && nEle + 1 <= SELECTOR_TO_ELEMS[BIT_TO_SELECTOR[nBit]]) { + if (pCmprsor->i_selector < BIT_TO_SELECTOR[nBit]) { + pCmprsor->i_selector = BIT_TO_SELECTOR[nBit]; + } + pCmprsor->i_end = (pCmprsor->i_end + 1) % 241; + pCmprsor->i_aZigzag[pCmprsor->i_end] = vZigzag; + pCmprsor->i_aBitN[pCmprsor->i_end] = nBit; + break; + } else { + while (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { + pCmprsor->i_selector++; + } + nEle = SELECTOR_TO_ELEMS[pCmprsor->i_selector]; + + if (pCmprsor->autoAlloc) { + code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + sizeof(uint64_t)); + if (code) return code; + } + + uint64_t *bp = (uint64_t *)(pCmprsor->aBuf[0] + pCmprsor->nBuf[0]); + pCmprsor->nBuf[0] += sizeof(uint64_t); + bp[0] = pCmprsor->i_selector; + uint8_t bits = BIT_PER_INTEGER[pCmprsor->i_selector]; + for (int32_t iVal = 0; iVal < nEle; iVal++) { + bp[0] |= ((pCmprsor->i_aZigzag[pCmprsor->i_start] & ((((uint64_t)1) << bits) - 1)) << (bits * iVal + 4)); + pCmprsor->i_start = (pCmprsor->i_start + 1) % 241; + } + + // reset and continue + pCmprsor->i_selector = 0; + for (int32_t iVal = pCmprsor->i_start; iVal < pCmprsor->i_end; iVal = (iVal + 1) % 241) { + if (pCmprsor->i_selector < BIT_TO_SELECTOR[pCmprsor->i_aBitN[iVal]]) { + pCmprsor->i_selector = BIT_TO_SELECTOR[pCmprsor->i_aBitN[iVal]]; + } + } + } + } + } else { + _copy_cmpr: + code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + nData); + if (code) return code; + + memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], pData, nData); + pCmprsor->nBuf[0] += nData; + } + pCmprsor->nVal++; + + return code; +} + +// Float ===================================================== +static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nData) { + int32_t code = 0; + + ASSERT(nData == sizeof(float)); + + union { + float f; + uint32_t u; + } val = {.f = *(float *)pData}; + + uint32_t diff = val.u ^ pCmprsor->f_prev; + pCmprsor->f_prev = val.u; + + int32_t clz, ctz; + if (diff) { + clz = BUILDIN_CLZ(diff); + ctz = BUILDIN_CTZ(diff); + } else { + clz = 32; + ctz = 32; + } + + uint8_t nBytes; + if (clz < ctz) { + nBytes = sizeof(uint32_t) - ctz / BITS_PER_BYTE; + if (nBytes) diff >>= (32 - nBytes * BITS_PER_BYTE); + } else { + nBytes = sizeof(uint32_t) - clz / BITS_PER_BYTE; + } + if (nBytes == 0) nBytes++; + + if ((pCmprsor->nVal & 0x1) == 0) { + if (pCmprsor->autoAlloc) { + code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 9); + if (code) return code; + } + + pCmprsor->f_flag_p = &pCmprsor->aBuf[0][pCmprsor->nBuf[0]]; + pCmprsor->nBuf[0]++; + + if (clz < ctz) { + pCmprsor->f_flag_p[0] = (0x08 | (nBytes - 1)); + } else { + pCmprsor->f_flag_p[0] = nBytes - 1; + } + } else { + if (clz < ctz) { + pCmprsor->f_flag_p[0] |= ((0x08 | (nBytes - 1)) << 4); + } else { + pCmprsor->f_flag_p[0] |= ((nBytes - 1) << 4); + } + } + for (; nBytes; nBytes--) { + pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (diff & 0xff); + pCmprsor->nBuf[0]++; + diff >>= BITS_PER_BYTE; + } + pCmprsor->nVal++; + + return code; +} + +// Double ===================================================== +static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nData) { + int32_t code = 0; + + ASSERT(nData == sizeof(double)); + + union { + double d; + uint64_t u; + } val = {.d = *(double *)pData}; + + uint64_t diff = val.u ^ pCmprsor->d_prev; + pCmprsor->d_prev = val.u; + + int32_t clz, ctz; + if (diff) { + clz = BUILDIN_CLZL(diff); + ctz = BUILDIN_CTZL(diff); + } else { + clz = 64; + ctz = 64; + } + + uint8_t nBytes; + if (clz < ctz) { + nBytes = sizeof(uint64_t) - ctz / BITS_PER_BYTE; + if (nBytes) diff >>= (64 - nBytes * BITS_PER_BYTE); + } else { + nBytes = sizeof(uint64_t) - clz / BITS_PER_BYTE; + } + if (nBytes == 0) nBytes++; + + if ((pCmprsor->nVal & 0x1) == 0) { + if (pCmprsor->autoAlloc) { + code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 17); + if (code) return code; + } + + pCmprsor->d_flag_p = &pCmprsor->aBuf[0][pCmprsor->nBuf[0]]; + pCmprsor->nBuf[0]++; + + if (clz < ctz) { + pCmprsor->d_flag_p[0] = (0x08 | (nBytes - 1)); + } else { + pCmprsor->d_flag_p[0] = nBytes - 1; + } + } else { + if (clz < ctz) { + pCmprsor->d_flag_p[0] |= ((0x08 | (nBytes - 1)) << 4); + } else { + pCmprsor->d_flag_p[0] |= ((nBytes - 1) << 4); + } + } + for (; nBytes; nBytes--) { + pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (diff & 0xff); + pCmprsor->nBuf[0]++; + diff >>= BITS_PER_BYTE; + } + pCmprsor->nVal++; + + return code; +} + +// Binary ===================================================== +static int32_t tCompBinary(SCompressor *pCmprsor, const void *pData, int32_t nData) { + int32_t code = 0; + + if (nData) { + if (pCmprsor->autoAlloc) { + code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + nData); + if (code) return code; + } + + memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], pData, nData); + pCmprsor->nBuf[0] += nData; + } + pCmprsor->nVal++; + + return code; +} + +// Bool ===================================================== +static const uint8_t BOOL_CMPR_TABLE[] = {0b01, 0b0100, 0b010000, 0b01000000}; + +static int32_t tCompBool(SCompressor *pCmprsor, const void *pData, int32_t nData) { + int32_t code = 0; + + bool vBool = *(int8_t *)pData; + + int32_t mod4 = pCmprsor->nVal & 3; + if (mod4 == 0) { + pCmprsor->nBuf[0]++; + + if (pCmprsor->autoAlloc) { + code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0]); + if (code) return code; + } + + pCmprsor->aBuf[0][pCmprsor->nBuf[0] - 1] = 0; + } + if (vBool) { + pCmprsor->aBuf[0][pCmprsor->nBuf[0] - 1] |= BOOL_CMPR_TABLE[mod4]; + } + pCmprsor->nVal++; + + return code; +} + +// SCompressor ===================================================== +int32_t tCompressorCreate(SCompressor **ppCmprsor) { + int32_t code = 0; + + *ppCmprsor = (SCompressor *)taosMemoryCalloc(1, sizeof(SCompressor)); + if ((*ppCmprsor) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + code = tRealloc(&(*ppCmprsor)->aBuf[0], 1024); + if (code) { + taosMemoryFree(*ppCmprsor); + *ppCmprsor = NULL; + goto _exit; + } + +_exit: + return code; +} + +int32_t tCompressorDestroy(SCompressor *pCmprsor) { + int32_t code = 0; + + if (pCmprsor) { + int32_t nBuf = sizeof(pCmprsor->aBuf) / sizeof(pCmprsor->aBuf[0]); + for (int32_t iBuf = 0; iBuf < nBuf; iBuf++) { + tFree(pCmprsor->aBuf[iBuf]); + } + + taosMemoryFree(pCmprsor); + } + + return code; +} + +int32_t tCompressorReset(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg, int8_t autoAlloc) { + int32_t code = 0; + + pCmprsor->type = type; + pCmprsor->cmprAlg = cmprAlg; + pCmprsor->autoAlloc = autoAlloc; + pCmprsor->nVal = 0; + + switch (type) { + case TSDB_DATA_TYPE_TIMESTAMP: + pCmprsor->ts_prev_val = 0; + pCmprsor->ts_prev_delta = 0; + pCmprsor->ts_flag_p = NULL; + pCmprsor->aBuf[0][0] = 1; // For timestamp, 1 means compressed, 0 otherwise + pCmprsor->nBuf[0] = 1; + break; + case TSDB_DATA_TYPE_BOOL: + pCmprsor->nBuf[0] = 0; + break; + case TSDB_DATA_TYPE_BINARY: + pCmprsor->nBuf[0] = 0; + break; + case TSDB_DATA_TYPE_FLOAT: + pCmprsor->f_prev = 0; + pCmprsor->f_flag_p = NULL; + pCmprsor->aBuf[0][0] = 0; // 0 means compressed, 1 otherwise (for backward compatibility) + pCmprsor->nBuf[0] = 1; + break; + case TSDB_DATA_TYPE_DOUBLE: + pCmprsor->d_prev = 0; + pCmprsor->d_flag_p = NULL; + pCmprsor->aBuf[0][0] = 0; // 0 means compressed, 1 otherwise (for backward compatibility) + pCmprsor->nBuf[0] = 1; + break; + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: + pCmprsor->i_prev = 0; + pCmprsor->i_selector = 0; + pCmprsor->i_start = 0; + pCmprsor->i_end = 0; + pCmprsor->aBuf[0][0] = 0; // 0 means compressed, 1 otherwise (for backward compatibility) + pCmprsor->nBuf[0] = 1; + break; + default: + break; + } + + return code; +} + +int32_t tCompGen(SCompressor *pCmprsor, const uint8_t **ppData, int64_t *nData) { + int32_t code = 0; + + if (pCmprsor->nVal == 0) { + *ppData = NULL; + *nData = 0; + return code; + } + + if (pCmprsor->cmprAlg == TWO_STAGE_COMP /*|| IS_VAR_DATA_TYPE(pCmprsor->type)*/) { + code = tRealloc(&pCmprsor->aBuf[1], pCmprsor->nBuf[0] + 1); + if (code) return code; + + int64_t ret = LZ4_compress_default(pCmprsor->aBuf[0], pCmprsor->aBuf[1] + 1, pCmprsor->nBuf[0], pCmprsor->nBuf[0]); + if (ret) { + pCmprsor->aBuf[1][0] = 0; + pCmprsor->nBuf[1] = ret + 1; + } else { + pCmprsor->aBuf[1][0] = 1; + memcpy(pCmprsor->aBuf[1] + 1, pCmprsor->aBuf[0], pCmprsor->nBuf[0]); + pCmprsor->nBuf[1] = pCmprsor->nBuf[0] + 1; + } + + *ppData = pCmprsor->aBuf[1]; + *nData = pCmprsor->nBuf[1]; + } else { + *ppData = pCmprsor->aBuf[0]; + *nData = pCmprsor->nBuf[0]; + } + + return code; +} + +int32_t tCompress(SCompressor *pCmprsor, const void *pData, int64_t nData) { + return DATA_TYPE_INFO[pCmprsor->type].cmprFn(pCmprsor, pData, nData); +} \ No newline at end of file diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 50c42ff170..1906a77127 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -621,6 +621,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_FETCH_MSG_MSSED_UP, "Rsma fetch msg is m TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_EMPTY_INFO, "Rsma info is empty") TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_SCHEMA, "Rsma invalid schema") TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_REGEX_MATCH, "Rsma regex match") +TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_STREAM_STATE_OPEN, "Rsma stream state open") +TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_STREAM_STATE_COMMIT, "Rsma stream state commit") //index TAOS_DEFINE_ERROR(TSDB_CODE_INDEX_REBUILDING, "Index is rebuilding") diff --git a/tests/script/sh/gpd.c b/tests/script/sh/gpd.c new file mode 100644 index 0000000000..8d69bacb5e --- /dev/null +++ b/tests/script/sh/gpd.c @@ -0,0 +1,74 @@ +#include +#include +#include +#ifdef LINUX +#include +#endif +#ifdef WINDOWS +#include +#endif +#include "taosudf.h" + +TAOS* taos = NULL; + +DLL_EXPORT int32_t gpd_init() { + taos = taos_connect("localhost", "root", "taosdata", "", 7100); + return 0; +} + +DLL_EXPORT int32_t gpd_destroy() { + taos_close(taos); + taos_cleanup(); + return 0; +} + +DLL_EXPORT int32_t gpd(SUdfDataBlock* block, SUdfColumn *resultCol) { + SUdfColumnMeta *meta = &resultCol->colMeta; + meta->bytes = 4; + meta->type = TSDB_DATA_TYPE_INT; + meta->scale = 0; + meta->precision = 0; + + SUdfColumnData *resultData = &resultCol->colData; + resultData->numOfRows = block->numOfRows; + for (int32_t i = 0; i < resultData->numOfRows; ++i) { + int j = 0; + for (; j < block->numOfCols; ++j) { + if (udfColDataIsNull(block->udfCols[j], i)) { + udfColDataSetNull(resultCol, i); + break; + } + } + if ( j == block->numOfCols) { + int32_t luckyNum = 88; + udfColDataSet(resultCol, i, (char *)&luckyNum, false); + } + } + TAOS_RES* res = taos_query(taos, "create database if not exists gpd"); + if (taos_errno(res) != 0) { + char* errstr = taos_errstr(res); + } + res = taos_query(taos, "create table gpd.st (ts timestamp, f int) tags(t int)"); + if (taos_errno(res) != 0) { + char* errstr = taos_errstr(res); + } + + taos_query(taos, "insert into gpd.t using gpd.st tags(1) values(now, 1) "); + if (taos_errno(res) != 0) { + char* errstr = taos_errstr(res); + } + + taos_query(taos, "select * from gpd.t"); + if (taos_errno(res) != 0) { + char* errstr = taos_errstr(res); + } + + //to simulate actual processing delay by udf +#ifdef LINUX + usleep(1 * 1000); // usleep takes sleep time in us (1 millionth of a second) +#endif +#ifdef WINDOWS + Sleep(1); +#endif + return 0; +} diff --git a/tests/script/tsim/query/udf.sim b/tests/script/tsim/query/udf.sim index 7f8b1044ef..0b48a815e2 100644 --- a/tests/script/tsim/query/udf.sim +++ b/tests/script/tsim/query/udf.sim @@ -144,18 +144,18 @@ if $data20 != 8.000000000 then return -1 endi -sql drop function bit_and; -sql show functions; -if $rows != 1 then - return -1 -endi -if $data00 != @l2norm@ then - return -1 - endi -sql drop function l2norm; -sql show functions; -if $rows != 0 then - return -1 -endi +#sql drop function bit_and; +#sql show functions; +#if $rows != 1 then +# return -1 +#endi +#if $data00 != @l2norm@ then +# return -1 +# endi +#sql drop function l2norm; +#sql show functions; +#if $rows != 0 then +# return -1 +#endi system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/system-test/7-tmq/tmq_taosx.py b/tests/system-test/7-tmq/tmq_taosx.py index 07602ec29f..626e935733 100644 --- a/tests/system-test/7-tmq/tmq_taosx.py +++ b/tests/system-test/7-tmq/tmq_taosx.py @@ -31,18 +31,20 @@ class TDTestCase: while True: dst = queryFile.readline() src = consumeFile.readline() - - if dst: + if src: if dst != src: - tdLog.exit("compare error: %s != %s"%src, dst) + tdLog.exit("compare error: %s != %s"%(src, dst)) else: break return - def checkDropData(self): + def checkDropData(self, drop): tdSql.execute('use db_taosx') tdSql.query("show tables") - tdSql.checkRows(2) + if drop: + tdSql.checkRows(10) + else: + tdSql.checkRows(15) tdSql.query("select * from jt order by i") tdSql.checkRows(2) tdSql.checkData(0, 1, 1) @@ -50,15 +52,72 @@ class TDTestCase: tdSql.checkData(0, 2, '{"k1":1,"k2":"hello"}') tdSql.checkData(1, 2, None) + tdSql.query("select * from sttb order by ts") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 13) + tdSql.checkData(1, 1, 16) + tdSql.checkData(0, 2, 22) + tdSql.checkData(1, 2, 25) + tdSql.checkData(0, 5, "sttb3") + tdSql.checkData(1, 5, "sttb4") + + tdSql.query("select * from stt order by ts") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 21) + tdSql.checkData(0, 2, 2) + tdSql.checkData(1, 2, 21) + tdSql.checkData(0, 5, "stt3") + tdSql.checkData(1, 5, "stt4") + tdSql.execute('use abc1') tdSql.query("show tables") - tdSql.checkRows(2) + if drop: + tdSql.checkRows(10) + else: + tdSql.checkRows(15) tdSql.query("select * from jt order by i") tdSql.checkRows(2) tdSql.checkData(0, 1, 1) tdSql.checkData(1, 1, 11) tdSql.checkData(0, 2, '{"k1":1,"k2":"hello"}') tdSql.checkData(1, 2, None) + + tdSql.query("select * from sttb order by ts") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 13) + tdSql.checkData(1, 1, 16) + tdSql.checkData(0, 2, 22) + tdSql.checkData(1, 2, 25) + tdSql.checkData(0, 5, "sttb3") + tdSql.checkData(1, 5, "sttb4") + + tdSql.query("select * from stt order by ts") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 21) + tdSql.checkData(0, 2, 2) + tdSql.checkData(1, 2, 21) + tdSql.checkData(0, 5, "stt3") + tdSql.checkData(1, 5, "stt4") + + return + + def checkDataTable(self): + tdSql.execute('use db_taosx') + tdSql.query("select * from meters_summary") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 120) + tdSql.checkData(0, 2, 1) + tdSql.checkData(0, 3, "San Francisco") + + tdSql.execute('use abc1') + tdSql.query("select * from meters_summary") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 120) + tdSql.checkData(0, 2, 1) + tdSql.checkData(0, 3, "San Francisco") + return def checkData(self): @@ -145,6 +204,19 @@ class TDTestCase: self.checkJson(cfgPath, "tmq_taosx_tmp") self.checkData() + self.checkDropData(False) + + return + + def checkWal1VgroupTable(self): + buildPath = tdCom.getBuildPath() + cfgPath = tdCom.getClientCfgPath() + cmdStr = '%s/build/bin/tmq_taosx_ci -c %s -sv 1 -dv 1 -t'%(buildPath, cfgPath) + tdLog.info(cmdStr) + os.system(cmdStr) + + self.checkJson(cfgPath, "tmq_taosx_tmp") + self.checkDataTable() return @@ -155,6 +227,7 @@ class TDTestCase: os.system(cmdStr) self.checkData() + self.checkDropData(False) return @@ -164,7 +237,7 @@ class TDTestCase: tdLog.info(cmdStr) os.system(cmdStr) - self.checkDropData() + self.checkDropData(True) return @@ -177,6 +250,19 @@ class TDTestCase: self.checkJson(cfgPath, "tmq_taosx_tmp_snapshot") self.checkData() + self.checkDropData(False) + + return + + def checkSnapshot1VgroupTable(self): + buildPath = tdCom.getBuildPath() + cfgPath = tdCom.getClientCfgPath() + cmdStr = '%s/build/bin/tmq_taosx_ci -c %s -sv 1 -dv 1 -s -t'%(buildPath, cfgPath) + tdLog.info(cmdStr) + os.system(cmdStr) + + self.checkJson(cfgPath, "tmq_taosx_tmp_snapshot") + self.checkDataTable() return @@ -187,6 +273,7 @@ class TDTestCase: os.system(cmdStr) self.checkData() + self.checkDropData(False) return @@ -196,7 +283,7 @@ class TDTestCase: tdLog.info(cmdStr) os.system(cmdStr) - self.checkDropData() + self.checkDropData(True) return @@ -205,6 +292,9 @@ class TDTestCase: self.checkWal1Vgroup() self.checkSnapshot1Vgroup() + self.checkWal1VgroupTable() + self.checkSnapshot1VgroupTable() + self.checkWalMultiVgroups() self.checkSnapshotMultiVgroups() diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 604add0eb3..f4a5c22d39 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -156,8 +156,8 @@ python3 ./test.py -f 2-query/sin.py python3 ./test.py -f 2-query/sin.py -R python3 ./test.py -f 2-query/smaTest.py python3 ./test.py -f 2-query/smaTest.py -R -#python3 ./test.py -f 2-query/sml.py -#python3 ./test.py -f 2-query/sml.py -R +python3 ./test.py -f 2-query/sml.py +python3 ./test.py -f 2-query/sml.py -R python3 ./test.py -f 2-query/spread.py python3 ./test.py -f 2-query/spread.py -R python3 ./test.py -f 2-query/sqrt.py @@ -512,6 +512,6 @@ python3 ./test.py -f 2-query/count_partition.py -Q 3 python3 ./test.py -f 2-query/max_partition.py -Q 3 python3 ./test.py -f 2-query/last_row.py -Q 3 python3 ./test.py -f 2-query/tsbsQuery.py -Q 3 -#python3 ./test.py -f 2-query/sml.py -Q 3 +python3 ./test.py -f 2-query/sml.py -Q 3 python3 ./test.py -f 2-query/interp.py -Q 3 diff --git a/utils/test/c/tmqSim.c b/utils/test/c/tmqSim.c index 71b31ba107..a043aa7a6d 100644 --- a/utils/test/c/tmqSim.c +++ b/utils/test/c/tmqSim.c @@ -653,23 +653,23 @@ static int32_t meta_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn int32_t code = tmq_get_raw(msg, &raw); if(code == TSDB_CODE_SUCCESS){ - int retCode = queryDB(pInfo->taos, "use metadb"); - if (retCode != 0) { - taosFprintfFile(g_fp, "error when use metadb\n"); - taosCloseFile(&g_fp); - exit(-1); - } - taosFprintfFile(g_fp, "raw:%p\n", &raw); - - tmq_write_raw(pInfo->taos, raw); +// int retCode = queryDB(pInfo->taos, "use metadb"); +// if (retCode != 0) { +// taosFprintfFile(g_fp, "error when use metadb\n"); +// taosCloseFile(&g_fp); +// exit(-1); +// } +// taosFprintfFile(g_fp, "raw:%p\n", &raw); +// +// tmq_write_raw(pInfo->taos, raw); } char* result = tmq_get_json_meta(msg); - if(result){ + if(result && strcmp(result, "") != 0){ //printf("meta result: %s\n", result); taosFprintfFile(pInfo->pConsumeMetaFile, "%s\n", result); - taosMemoryFree(result); } + tmq_free_json_meta(result); } totalRows++; @@ -818,8 +818,12 @@ void loop_consume(SThreadInfo* pInfo) { tmq_res_t msgType = tmq_get_res_type(tmqMsg); if (msgType == TMQ_RES_TABLE_META) { totalRows += meta_msg_process(tmqMsg, pInfo, totalMsgs); - } else if (msgType == TMQ_RES_DATA) - totalRows += data_msg_process(tmqMsg, pInfo, totalMsgs); + } else if (msgType == TMQ_RES_DATA){ + totalRows += data_msg_process(tmqMsg, pInfo, totalMsgs); + } else if (msgType == TMQ_RES_METADATA){ + meta_msg_process(tmqMsg, pInfo, totalMsgs); + totalRows += data_msg_process(tmqMsg, pInfo, totalMsgs); + } } taos_free_result(tmqMsg); diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index f917b9159e..3a598ba98b 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -54,24 +54,24 @@ static void msg_process(TAOS_RES* msg) { printf("db: %s\n", tmq_get_db_name(msg)); printf("vg: %d\n", tmq_get_vgroup_id(msg)); TAOS *pConn = use_db(); - if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META) { + if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META || tmq_get_res_type(msg) == TMQ_RES_METADATA) { char* result = tmq_get_json_meta(msg); if (result) { printf("meta result: %s\n", result); + if(g_fp && strcmp(result, "") != 0){ + taosFprintfFile(g_fp, result); + taosFprintfFile(g_fp, "\n"); + } } - if(g_fp){ - taosFprintfFile(g_fp, result); - taosFprintfFile(g_fp, "\n"); - } - tmq_free_json_meta(result); } tmq_raw_data raw = {0}; tmq_get_raw(msg, &raw); + printf("write raw data type: %d\n", raw.raw_type); int32_t ret = tmq_write_raw(pConn, raw); printf("write raw data: %s\n", tmq_err2str(ret)); - + tmq_free_raw(raw); taos_close(pConn); } @@ -309,6 +309,41 @@ int buildDatabase(TAOS* pConn, TAOS_RES* pRes){ } taos_free_result(pRes); } + + pRes = taos_query(pConn, + "create stable if not exists stt (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " + "nchar(8), t4 bool)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table stt, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, + "create stable if not exists sttb (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " + "nchar(8), t4 bool)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table sttb, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists stt1 using stt tags(2, \"stt1\", true) sttb1 using sttb tags(4, \"sttb1\", true) " + "stt2 using stt tags(43, \"stt2\", false) sttb2 using sttb tags(54, \"sttb2\", true)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table stt1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into stt3 using stt tags(23, \"stt3\", true) values(now + 1s, 1, 2, 'stt3') sttb3 using sttb tags(4, \"sttb3\", true) values(now + 2s, 13, 22, 'sttb3') " + "stt4 using stt tags(433, \"stt4\", false) values(now + 3s, 21, 21, 'stt4') sttb4 using sttb tags(543, \"sttb4\", true) values(now + 4s, 16, 25, 'sttb4')"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table stt1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + return 0; } @@ -542,47 +577,83 @@ void initLogFile() { } if(g_conf.snapShot){ - char *result[] = { - "{\"type\":\"create\",\"tableName\":\"st1\",\"tableType\":\"super\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":64},{\"name\":\"c4\",\"type\":5}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1},{\"name\":\"t2\",\"type\":8,\"length\":64}]}", - "{\"type\":\"create\",\"tableName\":\"ct0\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}", - "{\"type\":\"create\",\"tableName\":\"ct1\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2000}]}", - "{\"type\":\"create\",\"tableName\":\"ct2\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[]}", - "{\"type\":\"create\",\"tableName\":\"ct3\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":5000}]}", - "{\"type\":\"create\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c2\",\"type\":10,\"length\":8},{\"name\":\"cc3\",\"type\":5}],\"tags\":[]}", - "{\"type\":\"create\",\"tableName\":\"jt\",\"tableType\":\"super\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"i\",\"type\":4}],\"tags\":[{\"name\":\"t\",\"type\":15}]}", - "{\"type\":\"create\",\"tableName\":\"jt1\",\"tableType\":\"child\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[{\"name\":\"t\",\"type\":15,\"value\":\"{\\\"k1\\\":1,\\\"k2\\\":\\\"hello\\\"}\"}]}", - "{\"type\":\"create\",\"tableName\":\"jt2\",\"tableType\":\"child\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[]}", - }; - - for(int i = 0; i < sizeof(result)/sizeof(result[0]); i++){ - taosFprintfFile(pFile2, result[i]); - taosFprintfFile(pFile2, "\n"); + if(g_conf.subTable){ + char *result[] = { + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"meters_summary\",\"columns\":[{\"name\":\"_wstart\",\"type\":9},{\"name\":\"current\",\"type\":6},{\"name\":\"groupid\",\"type\":4},{\"name\":\"location\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"group_id\",\"type\":14}]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"t_d2a450ee819dcf7576f0282d9ac22dbc\",\"using\":\"meters_summary\",\"tagNum\":1,\"tags\":[{\"name\":\"group_id\",\"type\":14,\"value\":1.313555008277358e+19}],\"createList\":[]}" + }; + for(int i = 0; i < sizeof(result)/sizeof(result[0]); i++){ + taosFprintfFile(pFile2, result[i]); + taosFprintfFile(pFile2, "\n"); + } + }else{ + char *result[] = { + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"st1\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":64},{\"name\":\"c4\",\"type\":5}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1},{\"name\":\"t2\",\"type\":8,\"length\":64}]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct0\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct1\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2000}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct2\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct3\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":5000}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c2\",\"type\":10,\"length\":8},{\"name\":\"cc3\",\"type\":5}],\"tags\":[]}", + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"jt\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"i\",\"type\":4}],\"tags\":[{\"name\":\"t\",\"type\":15}]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"jt1\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[{\"name\":\"t\",\"type\":15,\"value\":\"{\\\"k1\\\":1,\\\"k2\\\":\\\"hello\\\"}\"}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"jt2\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"stt\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}", + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"sttb\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt1\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"sttb1\",\"using\":\"sttb\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":4},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"sttb1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt2\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":43},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt2\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":0}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"sttb2\",\"using\":\"sttb\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":54},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"sttb2\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt3\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":23},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt3\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"sttb3\",\"using\":\"sttb\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":4},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"sttb3\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt4\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":433},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt4\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":0}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"sttb4\",\"using\":\"sttb\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":543},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"sttb4\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}" + }; + for(int i = 0; i < sizeof(result)/sizeof(result[0]); i++){ + taosFprintfFile(pFile2, result[i]); + taosFprintfFile(pFile2, "\n"); + } } }else{ - char *result[] = { - "{\"type\":\"create\",\"tableName\":\"st1\",\"tableType\":\"super\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}", - "{\"type\":\"create\",\"tableName\":\"ct0\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}", - "{\"type\":\"create\",\"tableName\":\"ct1\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2000}]}", - "{\"type\":\"create\",\"tableName\":\"ct2\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[]}", - "{\"type\":\"create\",\"tableName\":\"ct3\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":3000}]}", - "{\"type\":\"alter\",\"tableName\":\"st1\",\"tableType\":\"super\",\"alterType\":5,\"colName\":\"c4\",\"colType\":5}", - "{\"type\":\"alter\",\"tableName\":\"st1\",\"tableType\":\"super\",\"alterType\":7,\"colName\":\"c3\",\"colType\":8,\"colLength\":64}", - "{\"type\":\"alter\",\"tableName\":\"st1\",\"tableType\":\"super\",\"alterType\":1,\"colName\":\"t2\",\"colType\":8,\"colLength\":64}", - "{\"type\":\"alter\",\"tableName\":\"ct3\",\"tableType\":\"child\",\"alterType\":4,\"colName\":\"t1\",\"colValue\":\"5000\",\"colValueNull\":false}", - "{\"type\":\"create\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":10,\"length\":4}],\"tags\":[]}", - "{\"type\":\"alter\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"alterType\":5,\"colName\":\"c3\",\"colType\":5}", - "{\"type\":\"alter\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"alterType\":7,\"colName\":\"c2\",\"colType\":10,\"colLength\":8}", - "{\"type\":\"alter\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"alterType\":10,\"colName\":\"c3\",\"colNewName\":\"cc3\"}", - "{\"type\":\"alter\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"alterType\":9}", - "{\"type\":\"alter\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"alterType\":6,\"colName\":\"c1\"}", - "{\"type\":\"create\",\"tableName\":\"jt\",\"tableType\":\"super\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"i\",\"type\":4}],\"tags\":[{\"name\":\"t\",\"type\":15}]}", - "{\"type\":\"create\",\"tableName\":\"jt1\",\"tableType\":\"child\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[{\"name\":\"t\",\"type\":15,\"value\":\"{\\\"k1\\\":1,\\\"k2\\\":\\\"hello\\\"}\"}]}", - "{\"type\":\"create\",\"tableName\":\"jt2\",\"tableType\":\"child\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[]}" - }; + if(g_conf.subTable){ + char *result[] = { + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"meters_summary\",\"columns\":[{\"name\":\"_wstart\",\"type\":9},{\"name\":\"current\",\"type\":6},{\"name\":\"groupid\",\"type\":4},{\"name\":\"location\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"group_id\",\"type\":14}]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"t_d2a450ee819dcf7576f0282d9ac22dbc\",\"using\":\"meters_summary\",\"tagNum\":1,\"tags\":[{\"name\":\"group_id\",\"type\":14,\"value\":1.313555008277358e+19}],\"createList\":[]}" + }; - for(int i = 0; i < sizeof(result)/sizeof(result[0]); i++){ - taosFprintfFile(pFile2, result[i]); - taosFprintfFile(pFile2, "\n"); + for(int i = 0; i < sizeof(result)/sizeof(result[0]); i++){ + taosFprintfFile(pFile2, result[i]); + taosFprintfFile(pFile2, "\n"); + } + }else{ + char *result[] = { + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"st1\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct0\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct1\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2000}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct2\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct3\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":3000}],\"createList\":[]}", + "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":5,\"colName\":\"c4\",\"colType\":5}", + "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":7,\"colName\":\"c3\",\"colType\":8,\"colLength\":64}", + "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":1,\"colName\":\"t2\",\"colType\":8,\"colLength\":64}", + "{\"type\":\"alter\",\"tableType\":\"child\",\"tableName\":\"ct3\",\"alterType\":4,\"colName\":\"t1\",\"colValue\":\"5000\",\"colValueNull\":false}", + "{\"type\":\"create\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":10,\"length\":4}],\"tags\":[]}", + "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":5,\"colName\":\"c3\",\"colType\":5}", + "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":7,\"colName\":\"c2\",\"colType\":10,\"colLength\":8}", + "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":10,\"colName\":\"c3\",\"colNewName\":\"cc3\"}", + "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":9}", + "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":6,\"colName\":\"c1\"}", + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"jt\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"i\",\"type\":4}],\"tags\":[{\"name\":\"t\",\"type\":15}]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"jt1\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[{\"name\":\"t\",\"type\":15,\"value\":\"{\\\"k1\\\":1,\\\"k2\\\":\\\"hello\\\"}\"}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"jt2\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"stt\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}", + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"sttb\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt1\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[{\"tableName\":\"stt1\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"sttb1\",\"using\":\"sttb\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":4},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"sttb1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"stt2\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":43},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt2\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":0}]},{\"tableName\":\"sttb2\",\"using\":\"sttb\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":54},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"sttb2\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt3\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":23},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt3\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[{\"tableName\":\"stt3\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":23},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt3\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"sttb3\",\"using\":\"sttb\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":4},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"sttb3\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"stt4\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":433},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt4\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":0}]},{\"tableName\":\"sttb4\",\"using\":\"sttb\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":543},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"sttb4\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}]}" + }; + + for(int i = 0; i < sizeof(result)/sizeof(result[0]); i++){ + taosFprintfFile(pFile2, result[i]); + taosFprintfFile(pFile2, "\n"); + } } } @@ -619,5 +690,6 @@ int main(int argc, char* argv[]) { tmq_t* tmq = build_consumer(); tmq_list_t* topic_list = build_topic_list(); basic_consume_loop(tmq, topic_list); + tmq_list_destroy(topic_list); taosCloseFile(&g_fp); }