From 1792bf5ff3221b3e148ca6a5db715f199f3342e7 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Sat, 7 May 2022 14:46:04 +0800 Subject: [PATCH 01/29] fix bug --- include/util/taoserror.h | 1 + source/client/inc/clientInt.h | 1 + source/client/src/clientStmt.c | 16 +++- source/dnode/vnode/src/vnd/vnodeSvr.c | 2 +- source/libs/scalar/src/filter.c | 16 ++++ source/libs/scalar/src/sclvector.c | 117 +++++++++++++++++++++++++- source/util/src/terror.c | 1 + tests/script/api/batchprepare.c | 6 +- 8 files changed, 148 insertions(+), 12 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 441e87eff7..d2c447e851 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -88,6 +88,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_CFG_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0114) #define TSDB_CODE_REPEAT_INIT TAOS_DEF_ERROR_CODE(0, 0x0115) #define TSDB_CODE_DUP_KEY TAOS_DEF_ERROR_CODE(0, 0x0116) +#define TSDB_CODE_NEED_RETRY TAOS_DEF_ERROR_CODE(0, 0x0117) #define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0140) #define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0141) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index ce5b101b4a..b021651c16 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -310,6 +310,7 @@ void hbMgrInitMqHbRspHandle(); SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery); int32_t getQueryPlan(SRequestObj* pRequest, SQuery* pQuery, SArray** pNodeList); int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList); +int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest); #ifdef __cplusplus } diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index f7ef1f7e81..d7b746605d 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -586,6 +586,16 @@ int stmtExec(TAOS_STMT *stmt) { STMT_ERR_RET(qBuildStmtOutput(pStmt->sql.pQuery, pStmt->exec.pVgHash, pStmt->exec.pBlockHash)); launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, TSDB_CODE_SUCCESS, true); } + + if (pStmt->exec.pRequest->code && NEED_CLIENT_HANDLE_ERROR(pStmt->exec.pRequest->code)) { + code = refreshMeta(pStmt->exec.pRequest->pTscObj, pStmt->exec.pRequest); + if (code) { + pStmt->exec.pRequest->code = code; + } else { + STMT_ERR_RET(stmtResetStmt(pStmt)); + STMT_ERR_RET(TSDB_CODE_NEED_RETRY); + } + } STMT_ERR_JRET(pStmt->exec.pRequest->code); @@ -613,13 +623,11 @@ int stmtClose(TAOS_STMT *stmt) { const char *stmtErrstr(TAOS_STMT *stmt) { STscStmt* pStmt = (STscStmt*)stmt; - if (stmt == NULL) { + if (stmt == NULL || NULL == pStmt->exec.pRequest) { return (char*) tstrerror(terrno); } - if (pStmt->exec.pRequest) { - pStmt->exec.pRequest->code = terrno; - } + pStmt->exec.pRequest->code = terrno; return taos_errstr(pStmt->exec.pRequest); } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 8460400b59..5f023fb637 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -537,7 +537,7 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in goto _exit; } - rsp.numOfRows += nRows; + rsp.affectedRows += nRows; } _exit: diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 80e5669cc2..eaab8e1f53 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3577,6 +3577,22 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { } if (QUERY_NODE_NODE_LIST == nodeType(*pNode)) { + SNodeListNode *listNode = (SNodeListNode *)*pNode; + if (QUERY_NODE_VALUE != nodeType(listNode->pNodeList->pHead->pNode)) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } + + SValueNode *valueNode = (SValueNode *)listNode->pNodeList->pHead->pNode; + uint8_t type = valueNode->node.resType.type; + SNode *node = NULL; + FOREACH(node, listNode->pNodeList) { + if (type != ((SValueNode *)node)->node.resType.type) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } + } + return DEAL_RES_CONTINUE; } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 1e37533f2c..b83147bfee 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -265,13 +265,53 @@ static FORCE_INLINE void varToTimestamp(char *buf, SScalarParam* pOut, int32_t r } static FORCE_INLINE void varToSigned(char *buf, SScalarParam* pOut, int32_t rowIndex) { - int64_t value = strtoll(buf, NULL, 10); - colDataAppendInt64(pOut->columnData, rowIndex, &value); + switch (pOut->columnData->info.type) { + case TSDB_DATA_TYPE_TINYINT: { + int8_t value = (int8_t)strtoll(buf, NULL, 10); + colDataAppendInt8(pOut->columnData, rowIndex, (int8_t*)&value); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + int16_t value = (int16_t)strtoll(buf, NULL, 10); + colDataAppendInt16(pOut->columnData, rowIndex, (int16_t*)&value); + break; + } + case TSDB_DATA_TYPE_INT: { + int32_t value = (int32_t)strtoll(buf, NULL, 10); + colDataAppendInt32(pOut->columnData, rowIndex, (int32_t*)&value); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + int64_t value = (int64_t)strtoll(buf, NULL, 10); + colDataAppendInt64(pOut->columnData, rowIndex, (int64_t*)&value); + break; + } + } } static FORCE_INLINE void varToUnsigned(char *buf, SScalarParam* pOut, int32_t rowIndex) { - uint64_t value = strtoull(buf, NULL, 10); - colDataAppendInt64(pOut->columnData, rowIndex, (int64_t*) &value); + switch (pOut->columnData->info.type) { + case TSDB_DATA_TYPE_UTINYINT: { + uint8_t value = (uint8_t)strtoull(buf, NULL, 10); + colDataAppendInt8(pOut->columnData, rowIndex, (int8_t*)&value); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + uint16_t value = (uint16_t)strtoull(buf, NULL, 10); + colDataAppendInt16(pOut->columnData, rowIndex, (int16_t*)&value); + break; + } + case TSDB_DATA_TYPE_UINT: { + uint32_t value = (uint32_t)strtoull(buf, NULL, 10); + colDataAppendInt32(pOut->columnData, rowIndex, (int32_t*)&value); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + uint64_t value = (uint64_t)strtoull(buf, NULL, 10); + colDataAppendInt64(pOut->columnData, rowIndex, (int64_t*)&value); + break; + } + } } static FORCE_INLINE void varToFloat(char *buf, SScalarParam* pOut, int32_t rowIndex) { @@ -453,6 +493,71 @@ void convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t } } +int32_t vectorConvertToVarData(const SScalarParam* pIn, SScalarParam* pOut, int16_t inType, int16_t outType) { + SColumnInfoData* pInputCol = pIn->columnData; + SColumnInfoData* pOutputCol = pOut->columnData; + char tmp[128] = {0}; + + if (IS_SIGNED_NUMERIC_TYPE(inType) || inType == TSDB_DATA_TYPE_BOOL || inType == TSDB_DATA_TYPE_TIMESTAMP) { + for (int32_t i = 0; i < pIn->numOfRows; ++i) { + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + colDataAppendNULL(pOutputCol, i); + continue; + } + + int64_t value = 0; + GET_TYPED_DATA(value, int64_t, inType, colDataGetData(pInputCol, i)); + int32_t len = sprintf(varDataVal(tmp), "%" PRId64, value); + varDataLen(tmp) = len; + if (outType == TSDB_DATA_TYPE_NCHAR) { + varToNchar(tmp, pOut, i); + } else { + colDataAppend(pOutputCol, i, (char *)&value, false); + } + } + } else if (IS_UNSIGNED_NUMERIC_TYPE(inType)) { + for (int32_t i = 0; i < pIn->numOfRows; ++i) { + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + colDataAppendNULL(pOutputCol, i); + continue; + } + + uint64_t value = 0; + GET_TYPED_DATA(value, uint64_t, inType, colDataGetData(pInputCol, i)); + int32_t len = sprintf(varDataVal(tmp), "%" PRIu64, value); + varDataLen(tmp) = len; + if (outType == TSDB_DATA_TYPE_NCHAR) { + varToNchar(tmp, pOut, i); + } else { + colDataAppend(pOutputCol, i, (char *)&value, false); + } + } + } else if (IS_FLOAT_TYPE(inType)) { + for (int32_t i = 0; i < pIn->numOfRows; ++i) { + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + colDataAppendNULL(pOutputCol, i); + continue; + } + + double value = 0; + GET_TYPED_DATA(value, double, inType, colDataGetData(pInputCol, i)); + int32_t len = sprintf(varDataVal(tmp), "%lf", value); + varDataLen(tmp) = len; + if (outType == TSDB_DATA_TYPE_NCHAR) { + varToNchar(tmp, pOut, i); + } else { + colDataAppend(pOutputCol, i, (char *)&value, false); + } + } + } else { + sclError("not supported input type:%d", inType); + return TSDB_CODE_QRY_APP_ERROR; + } + + return TSDB_CODE_SUCCESS; +} + + // TODO opt performance int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut) { SColumnInfoData* pInputCol = pIn->columnData; @@ -610,6 +715,10 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut) { } break; } + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: { + return vectorConvertToVarData(pIn, pOut, inType, outType); + } default: sclError("invalid convert output type:%d", outType); return TSDB_CODE_QRY_APP_ERROR; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 1470496c68..60af8c241c 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -94,6 +94,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MSG_NOT_PROCESSED, "Message not processed TAOS_DEFINE_ERROR(TSDB_CODE_CFG_NOT_FOUND, "Config not found") TAOS_DEFINE_ERROR(TSDB_CODE_REPEAT_INIT, "Repeat initialization") TAOS_DEFINE_ERROR(TSDB_CODE_DUP_KEY, "Cannot add duplicate keys to hash") +TAOS_DEFINE_ERROR(TSDB_CODE_NEED_RETRY, "Retry needed") TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, "Ref out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, "too many Ref Objs") diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 99e546fcd0..beabb0c473 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -11,7 +11,7 @@ int32_t shortColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_INT}; int32_t fullColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_BOOL, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_UTINYINT, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_USMALLINT, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_UINT, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_UBIGINT, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_BINARY, TSDB_DATA_TYPE_NCHAR}; -int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_DOUBLE}; +int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_NCHAR}; int32_t optrIdxList[] = {0, 1, 2}; typedef struct { @@ -235,7 +235,7 @@ CaseCtrl gCaseCtrl = { // query case with specified col&oper .bindColTypeList = NULL, .optrIdxListNum = 0, .optrIdxList = NULL, - .checkParamNum = true, + .checkParamNum = false, .printRes = true, .runTimes = 0, .caseRunIdx = -1, @@ -795,7 +795,7 @@ int32_t prepareQueryCondData(BindData *data, int32_t tblIdx) { } } - generateQuerySQL(data, tblIdx); + generateQueryCondSQL(data, tblIdx); return 0; } From 6f49a2ca73964a43067c65fc1ebdbd17ae9a8e23 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Sat, 7 May 2022 21:10:52 +0800 Subject: [PATCH 02/29] stmt insert --- include/libs/qcom/query.h | 2 +- source/client/inc/clientStmt.h | 1 + source/client/src/clientSml.c | 2 +- source/client/src/clientStmt.c | 22 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 4 +- source/libs/catalog/inc/catalogInt.h | 2 +- source/libs/parser/src/parInsert.c | 3 +- source/libs/parser/src/parTranslater.c | 2 +- tests/script/api/batchprepare.c | 524 +++++++++++++++++-------- 9 files changed, 394 insertions(+), 168 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index c1e53fa805..3cc71546fc 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -182,7 +182,7 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #define SET_META_TYPE_TABLE(t) (t) = META_TYPE_TABLE #define SET_META_TYPE_BOTH_TABLE(t) (t) = META_TYPE_BOTH_TABLE -#define NEED_CLIENT_RM_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_INVALID_TABLE_ID || (_code) == TSDB_CODE_VND_TB_NOT_EXIST) +#define NEED_CLIENT_RM_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_PAR_TABLE_NOT_EXIST || (_code) == TSDB_CODE_VND_TB_NOT_EXIST) #define NEED_CLIENT_REFRESH_VG_ERROR(_code) ((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID) #define NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_TABLE_RECREATED) #define NEED_CLIENT_HANDLE_ERROR(_code) (NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code)) diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index 43e886faf5..32da75fb1e 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -60,6 +60,7 @@ typedef struct SStmtBindInfo { int32_t sBindRowNum; int32_t sBindLastIdx; int8_t tbType; + bool tagsCached; void* boundTags; char* tbName; SName sname; diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 85cd0661e3..a5b7b08d0a 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -432,7 +432,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle* info) { code = catalogGetSTableMeta(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, &pTableMeta); - if (code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_MND_INVALID_STB) { + if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_MND_INVALID_STB) { SSchemaAction schemaAction = {0}; schemaAction.action = SCHEMA_ACTION_CREATE_STABLE; memcpy(schemaAction.createSTable.sTableName, superTable, superTableLen); diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index ba8123d960..347debe68d 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -123,6 +123,7 @@ int32_t stmtSetBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags) { pStmt->bInfo.tbSuid = pTableMeta->suid; pStmt->bInfo.tbType = pTableMeta->tableType; pStmt->bInfo.boundTags = tags; + pStmt->bInfo.tagsCached = false; return TSDB_CODE_SUCCESS; } @@ -207,8 +208,6 @@ int32_t stmtParseSql(STscStmt* pStmt) { STMT_ERR_RET(TSDB_CODE_TSC_STMT_CLAUSE_ERROR); } - STMT_ERR_RET(stmtCacheBlock(pStmt)); - return TSDB_CODE_SUCCESS; } @@ -219,8 +218,10 @@ int32_t stmtCleanBindInfo(STscStmt* pStmt) { pStmt->bInfo.needParse = true; taosMemoryFreeClear(pStmt->bInfo.tbName); - destroyBoundColumnInfo(pStmt->bInfo.boundTags); - taosMemoryFreeClear(pStmt->bInfo.boundTags); + if (!pStmt->bInfo.tagsCached) { + destroyBoundColumnInfo(pStmt->bInfo.boundTags); + taosMemoryFreeClear(pStmt->bInfo.boundTags); + } return TSDB_CODE_SUCCESS; } @@ -275,6 +276,7 @@ int32_t stmtCleanSQLInfo(STscStmt* pStmt) { qDestroyStmtDataBlock(pCache->pDataBlock); destroyBoundColumnInfo(pCache->boundTags); + taosMemoryFreeClear(pCache->boundTags); pIter = taosHashIterate(pStmt->sql.pTableCache, pIter); } @@ -302,7 +304,15 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { STableMeta *pTableMeta = NULL; SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp); - STMT_ERR_RET(catalogGetTableMeta(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->bInfo.sname, &pTableMeta)); + int32_t code = catalogGetTableMeta(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->bInfo.sname, &pTableMeta); + if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code) { + STMT_ERR_RET(stmtCleanBindInfo(pStmt)); + + return TSDB_CODE_SUCCESS; + } + + STMT_ERR_RET(code); + uint64_t uid = pTableMeta->uid; uint64_t suid = pTableMeta->suid; int8_t tableType = pTableMeta->tableType; @@ -328,6 +338,7 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { pStmt->bInfo.tbSuid = suid; pStmt->bInfo.tbType = tableType; pStmt->bInfo.boundTags = pCache->boundTags; + pStmt->bInfo.tagsCached = true; return TSDB_CODE_SUCCESS; } @@ -340,6 +351,7 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { pStmt->bInfo.tbSuid = suid; pStmt->bInfo.tbType = tableType; pStmt->bInfo.boundTags = pCache->boundTags; + pStmt->bInfo.tagsCached = true; STableDataBlocks* pNewBlock = NULL; STMT_ERR_RET(qRebuildStmtDataBlock(&pNewBlock, pCache->pDataBlock)); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 0b9f21dbcf..3d9ef6cf8a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3869,7 +3869,7 @@ int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const ch if (metaGetTableEntryByUid(&mr, uid) < 0) { tsdbError("%p failed to get stable, uid:%" PRIu64 ", TID:0x%" PRIx64 " QID:0x%" PRIx64, pMeta, uid, taskId, reqId); - terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; + terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; goto _error; } else { tsdbDebug("%p succeed to get stable, uid:%" PRIu64 ", TID:0x%" PRIx64 " QID:0x%" PRIx64, pMeta, uid, taskId, reqId); @@ -3939,7 +3939,7 @@ int32_t tsdbGetOneTableGroup(void* pMeta, uint64_t uid, TSKEY startKey, STableGr metaReaderInit(&mr, (SMeta*)pMeta, 0); if (metaGetTableEntryByUid(&mr, uid) < 0) { - terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; + terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; goto _error; } diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 3e8528e3d9..1bf21ad7d1 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -36,7 +36,7 @@ extern "C" { #define CTG_DEFAULT_INVALID_VERSION (-1) -#define CTG_ERR_CODE_TABLE_NOT_EXIST TSDB_CODE_TDB_INVALID_TABLE_ID +#define CTG_ERR_CODE_TABLE_NOT_EXIST TSDB_CODE_PAR_TABLE_NOT_EXIST enum { CTG_READ = 1, diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 05d247c037..4bc04f8eab 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1151,6 +1151,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { (*pCxt->pStmtCb->setExecInfoFn)(pCxt->pStmtCb->pStmt, pCxt->pVgroupsHashObj, pCxt->pTableBlockHashObj); pCxt->pVgroupsHashObj = NULL; pCxt->pTableBlockHashObj = NULL; + pCxt->pTableMeta = NULL; return TSDB_CODE_SUCCESS; } @@ -1276,7 +1277,7 @@ int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, char *tN return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SSchema* pSchema = getTableTagSchema(pDataBlock->pTableMeta); + SSchema* pSchema = pDataBlock->pTableMeta->schema; SKvParam param = {.builder = &tagBuilder}; for (int c = 0; c < tags->numOfBound; ++c) { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 6874b5b7d4..ec593c75ac 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2574,7 +2574,7 @@ static int32_t translateDropTable(STranslateContext* pCxt, SDropTableStmt* pStmt SName tableName; int32_t code = getTableMetaImpl( pCxt, toName(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, &tableName), &pTableMeta); - if ((TSDB_CODE_TDB_INVALID_TABLE_ID == code || TSDB_CODE_VND_TB_NOT_EXIST == code) && pClause->ignoreNotExists) { + if ((TSDB_CODE_PAR_TABLE_NOT_EXIST == code || TSDB_CODE_VND_TB_NOT_EXIST == code) && pClause->ignoreNotExists) { return TSDB_CODE_SUCCESS; } if (TSDB_CODE_SUCCESS == code) { diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index beabb0c473..3dddb03f3b 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -20,6 +20,11 @@ typedef struct { bool enclose; } OperInfo; +typedef enum { + BP_BIND_TAG = 1, + BP_BIND_COL, +} BP_BIND_TYPE; + OperInfo operInfo[] = { {">", 2, false}, {">=", 2, false}, @@ -41,6 +46,8 @@ int32_t varoperatorList[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; char *bpStbPrefix = "st"; char *bpTbPrefix = "t"; +int32_t bpDefaultStbId = 1; + //char *operatorList[] = {">", ">=", "<", "<=", "=", "<>", "in", "not in"}; @@ -65,6 +72,7 @@ typedef struct { char* isNull; int32_t* binaryLen; TAOS_MULTI_BIND* pBind; + TAOS_MULTI_BIND* pTags; char* sql; int32_t* colTypes; int32_t colNum; @@ -83,6 +91,7 @@ int insertMBMETest2(TAOS_STMT *stmt, TAOS *taos); int insertMBMETest3(TAOS_STMT *stmt, TAOS *taos); int insertMBMETest4(TAOS_STMT *stmt, TAOS *taos); int insertMPMETest1(TAOS_STMT *stmt, TAOS *taos); +int insertAUTOTest1(TAOS_STMT *stmt, TAOS *taos); int querySUBTTest1(TAOS_STMT *stmt, TAOS *taos); enum { @@ -102,45 +111,48 @@ typedef struct { int32_t rowNum; int32_t bindRowNum; int32_t bindColNum; // equal colNum in full column case + int32_t bindTagNum; // equal colNum in full column case int32_t bindNullNum; int32_t runTimes; int32_t preCaseIdx; } CaseCfg; CaseCfg gCase[] = { - {"insert:MBSE1-FULL", tListLen(shortColList), shortColList, TTYPE_INSERT, false, true, insertMBSETest1, 1, 10, 10, 0, 0, 1, -1}, - {"insert:MBSE1-FULL", tListLen(shortColList), shortColList, TTYPE_INSERT, false, true, insertMBSETest1, 10, 100, 10, 0, 0, 1, -1}, + {"insert:MBSE1-FULL", tListLen(shortColList), shortColList, TTYPE_INSERT, false, true, insertMBSETest1, 1, 10, 10, 0, 0, 0, 1, -1}, + {"insert:MBSE1-FULL", tListLen(shortColList), shortColList, TTYPE_INSERT, false, true, insertMBSETest1, 10, 100, 10, 0, 0, 0, 1, -1}, - {"insert:MBSE1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBSETest1, 10, 10, 2, 0, 0, 1, -1}, - {"insert:MBSE1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBSETest1, 10, 10, 2, 12, 0, 1, -1}, - {"insert:MBSE1-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBSETest1, 10, 10, 2, 2, 0, 1, -1}, + {"insert:MBSE1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBSETest1, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:MBSE1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBSETest1, 10, 10, 2, 12, 0, 0, 1, -1}, + {"insert:MBSE1-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBSETest1, 10, 10, 2, 2, 0, 0, 1, -1}, - {"insert:MBSE2-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBSETest2, 10, 10, 2, 0, 0, 1, -1}, - {"insert:MBSE2-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBSETest2, 10, 10, 2, 12, 0, 1, -1}, - {"insert:MBSE2-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBSETest2, 10, 10, 2, 2, 0, 1, -1}, + {"insert:MBSE2-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBSETest2, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:MBSE2-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBSETest2, 10, 10, 2, 12, 0, 0, 1, -1}, + {"insert:MBSE2-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBSETest2, 10, 10, 2, 2, 0, 0, 1, -1}, - {"insert:MBME1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBMETest1, 10, 10, 2, 0, 0, 1, -1}, - {"insert:MBME1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest1, 10, 10, 2, 12, 0, 1, -1}, - {"insert:MBME1-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest1, 10, 10, 2, 2, 0, 1, -1}, + {"insert:MBME1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBMETest1, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:MBME1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest1, 10, 10, 2, 12, 0, 0, 1, -1}, + {"insert:MBME1-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest1, 10, 10, 2, 2, 0, 0, 1, -1}, // 11 - {"insert:MBME2-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBMETest2, 10, 10, 2, 0, 0, 1, -1}, - {"insert:MBME2-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest2, 10, 10, 2, 12, 0, 1, -1}, - {"insert:MBME2-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest2, 10, 10, 2, 2, 0, 1, -1}, + {"insert:MBME2-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBMETest2, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:MBME2-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest2, 10, 10, 2, 12, 0, 0, 1, -1}, + {"insert:MBME2-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest2, 10, 10, 2, 2, 0, 0, 1, -1}, - {"insert:MBME3-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBMETest3, 10, 10, 2, 0, 0, 1, -1}, - {"insert:MBME3-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest3, 10, 10, 2, 12, 0, 1, -1}, - {"insert:MBME3-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest3, 10, 10, 2, 2, 0, 1, -1}, + {"insert:MBME3-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBMETest3, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:MBME3-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest3, 10, 10, 2, 12, 0, 0, 1, -1}, + {"insert:MBME3-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest3, 10, 10, 2, 2, 0, 0, 1, -1}, - {"insert:MBME4-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBMETest4, 10, 10, 2, 0, 0, 1, -1}, - {"insert:MBME4-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest4, 10, 10, 2, 12, 0, 1, -1}, - {"insert:MBME4-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest4, 10, 10, 2, 2, 0, 1, -1}, + {"insert:MBME4-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBMETest4, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:MBME4-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest4, 10, 10, 2, 12, 0, 0, 1, -1}, + {"insert:MBME4-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest4, 10, 10, 2, 2, 0, 0, 1, -1}, - {"insert:MPME1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMPMETest1, 10, 10, 2, 0, 0, 1, -1}, - {"insert:MPME1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMPMETest1, 10, 10, 2, 12, 0, 1, -1}, + {"insert:MPME1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMPMETest1, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:MPME1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMPMETest1, 10, 10, 2, 12, 0, 0, 1, -1}, // 22 - {"query:SUBT-FULL", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, querySUBTTest1, 10, 10, 1, 3, 0, 1, 2}, + {"insert:AUTO1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, true, true, insertAUTOTest1, 10, 10, 2, 0, 0, 0, 1, -1}, + + {"query:SUBT-FULL", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, querySUBTTest1, 10, 10, 1, 3, 0, 0, 1, 2}, }; @@ -149,7 +161,6 @@ CaseCfg *gCurCase = NULL; typedef struct { char caseCatalog[255]; int32_t bindNullNum; - bool prepareStb; bool checkParamNum; bool printRes; bool printCreateTblSql; @@ -157,9 +168,12 @@ typedef struct { bool printStmtSql; int32_t rowNum; //row num for one table int32_t bindColNum; + int32_t bindTagNum; int32_t bindRowNum; //row num for once bind int32_t bindColTypeNum; int32_t* bindColTypeList; + int32_t bindTagTypeNum; + int32_t* bindTagTypeList; int32_t optrIdxListNum; int32_t* optrIdxList; int32_t runTimes; @@ -172,15 +186,17 @@ typedef struct { #if 0 CaseCtrl gCaseCtrl = { // default .bindNullNum = 0, - .prepareStb = false, .printCreateTblSql = false, .printQuerySql = true, .printStmtSql = true, .rowNum = 0, .bindColNum = 0, + .bindTagNum = 0, .bindRowNum = 0, .bindColTypeNum = 0, .bindColTypeList = NULL, + .bindTagTypeNum = 0, + .bindTagTypeList = NULL, .optrIdxListNum = 0, .optrIdxList = NULL, .checkParamNum = false, @@ -196,7 +212,6 @@ CaseCtrl gCaseCtrl = { // default #if 0 CaseCtrl gCaseCtrl = { // query case with specified col&oper .bindNullNum = 0, - .prepareStb = false, .printCreateTblSql = false, .printQuerySql = true, .printStmtSql = true, @@ -224,8 +239,7 @@ CaseCtrl gCaseCtrl = { // query case with specified col&oper #if 1 CaseCtrl gCaseCtrl = { // query case with specified col&oper .bindNullNum = 0, - .prepareStb = false, - .printCreateTblSql = false, + .printCreateTblSql = true, .printQuerySql = true, .printStmtSql = true, .rowNum = 0, @@ -239,10 +253,10 @@ CaseCtrl gCaseCtrl = { // query case with specified col&oper .printRes = true, .runTimes = 0, .caseRunIdx = -1, - .optrIdxListNum = tListLen(optrIdxList), - .optrIdxList = optrIdxList, - .bindColTypeNum = tListLen(bindColTypeList), - .bindColTypeList = bindColTypeList, + //.optrIdxListNum = tListLen(optrIdxList), + //.optrIdxList = optrIdxList, + //.bindColTypeNum = tListLen(bindColTypeList), + //.bindColTypeList = bindColTypeList, .caseIdx = 22, .caseNum = 1, .caseRunNum = 1, @@ -299,6 +313,78 @@ void generateInsertSQL(BindData *data) { } else { len = sprintf(data->sql, "insert into %s0 ", bpTbPrefix); } + + if (gCurCase->bindTagNum > 0) { + len += sprintf(data->sql + len, "using %s%d ", bpStbPrefix, bpDefaultStbId); + + if (!gCurCase->fullCol) { + len += sprintf(data->sql + len, "("); + for (int c = 0; c < gCurCase->bindTagNum; ++c) { + if (c) { + len += sprintf(data->sql + len, ","); + } + switch (data->pTags[c].buffer_type) { + case TSDB_DATA_TYPE_BOOL: + len += sprintf(data->sql + len, "tbooldata"); + break; + case TSDB_DATA_TYPE_TINYINT: + len += sprintf(data->sql + len, "ttinydata"); + break; + case TSDB_DATA_TYPE_SMALLINT: + len += sprintf(data->sql + len, "tsmalldata"); + break; + case TSDB_DATA_TYPE_INT: + len += sprintf(data->sql + len, "tintdata"); + break; + case TSDB_DATA_TYPE_BIGINT: + len += sprintf(data->sql + len, "tbigdata"); + break; + case TSDB_DATA_TYPE_FLOAT: + len += sprintf(data->sql + len, "tfloatdata"); + break; + case TSDB_DATA_TYPE_DOUBLE: + len += sprintf(data->sql + len, "tdoubledata"); + break; + case TSDB_DATA_TYPE_VARCHAR: + len += sprintf(data->sql + len, "tbinarydata"); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + len += sprintf(data->sql + len, "tts"); + break; + case TSDB_DATA_TYPE_NCHAR: + len += sprintf(data->sql + len, "tnchardata"); + break; + case TSDB_DATA_TYPE_UTINYINT: + len += sprintf(data->sql + len, "tutinydata"); + break; + case TSDB_DATA_TYPE_USMALLINT: + len += sprintf(data->sql + len, "tusmalldata"); + break; + case TSDB_DATA_TYPE_UINT: + len += sprintf(data->sql + len, "tuintdata"); + break; + case TSDB_DATA_TYPE_UBIGINT: + len += sprintf(data->sql + len, "tubigdata"); + break; + default: + printf("!!!invalid tag type:%d", data->pTags[c].buffer_type); + exit(1); + } + } + + len += sprintf(data->sql + len, ") "); + } + + len += sprintf(data->sql + len, "tags ("); + for (int c = 0; c < gCurCase->bindTagNum; ++c) { + if (c) { + len += sprintf(data->sql + len, ","); + } + len += sprintf(data->sql + len, "?"); + } + len += sprintf(data->sql + len, ") "); + } + if (!gCurCase->fullCol) { len += sprintf(data->sql + len, "("); for (int c = 0; c < gCurCase->bindColNum; ++c) { @@ -552,7 +638,7 @@ void generateErrorSQL(BindData *data, int32_t tblIdx) { } } -void generateDataType(BindData *data, int32_t bindIdx, int32_t colIdx, int32_t *dataType) { +void generateColDataType(BindData *data, int32_t bindIdx, int32_t colIdx, int32_t *dataType) { if (bindIdx < gCurCase->bindColNum) { if (gCaseCtrl.bindColTypeNum) { *dataType = gCaseCtrl.bindColTypeList[colIdx]; @@ -584,104 +670,146 @@ void generateDataType(BindData *data, int32_t bindIdx, int32_t colIdx, int32_t * } } -int32_t prepareColData(BindData *data, int32_t bindIdx, int32_t rowIdx, int32_t colIdx) { - int32_t dataType = TSDB_DATA_TYPE_TIMESTAMP; +void generateTagDataType(BindData *data, int32_t bindIdx, int32_t colIdx, int32_t *dataType) { + if (bindIdx < gCurCase->bindTagNum) { + if (gCaseCtrl.bindTagTypeNum) { + *dataType = gCaseCtrl.bindTagTypeList[colIdx]; + return; + } else if (gCurCase->fullCol) { + *dataType = gCurCase->colList[bindIdx]; + return; + } else { + while (true) { + *dataType = rand() % (TSDB_DATA_TYPE_MAX - 1) + 1; + if (*dataType == TSDB_DATA_TYPE_JSON || *dataType == TSDB_DATA_TYPE_DECIMAL + || *dataType == TSDB_DATA_TYPE_BLOB || *dataType == TSDB_DATA_TYPE_MEDIUMBLOB + || *dataType == TSDB_DATA_TYPE_VARBINARY) { + continue; + } + + if (colExists(data->pTags, *dataType)) { + continue; + } + + break; + } + } + } else { + *dataType = data->pTags[bindIdx%gCurCase->bindTagNum].buffer_type; + } +} + + +int32_t prepareColData(BP_BIND_TYPE bType, BindData *data, int32_t bindIdx, int32_t rowIdx, int32_t colIdx) { + int32_t dataType = TSDB_DATA_TYPE_TIMESTAMP; + TAOS_MULTI_BIND *pBase = NULL; + + if (bType == BP_BIND_TAG) { + pBase = data->pTags; + generateTagDataType(data, bindIdx, colIdx, &dataType); + } else { + pBase = data->pBind; + generateColDataType(data, bindIdx, colIdx, &dataType); + } - generateDataType(data, bindIdx, colIdx, &dataType); switch (dataType) { case TSDB_DATA_TYPE_BOOL: - data->pBind[bindIdx].buffer_length = sizeof(bool); - data->pBind[bindIdx].buffer = data->boolData + rowIdx; - data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + pBase[bindIdx].buffer_length = sizeof(bool); + pBase[bindIdx].buffer = data->boolData + rowIdx; + pBase[bindIdx].length = NULL; + pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_TINYINT: - data->pBind[bindIdx].buffer_length = sizeof(int8_t); - data->pBind[bindIdx].buffer = data->tinyData + rowIdx; - data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + pBase[bindIdx].buffer_length = sizeof(int8_t); + pBase[bindIdx].buffer = data->tinyData + rowIdx; + pBase[bindIdx].length = NULL; + pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_SMALLINT: - data->pBind[bindIdx].buffer_length = sizeof(int16_t); - data->pBind[bindIdx].buffer = data->smallData + rowIdx; - data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + pBase[bindIdx].buffer_length = sizeof(int16_t); + pBase[bindIdx].buffer = data->smallData + rowIdx; + pBase[bindIdx].length = NULL; + pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_INT: - data->pBind[bindIdx].buffer_length = sizeof(int32_t); - data->pBind[bindIdx].buffer = data->intData + rowIdx; - data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + pBase[bindIdx].buffer_length = sizeof(int32_t); + pBase[bindIdx].buffer = data->intData + rowIdx; + pBase[bindIdx].length = NULL; + pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_BIGINT: - data->pBind[bindIdx].buffer_length = sizeof(int64_t); - data->pBind[bindIdx].buffer = data->bigData + rowIdx; - data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + pBase[bindIdx].buffer_length = sizeof(int64_t); + pBase[bindIdx].buffer = data->bigData + rowIdx; + pBase[bindIdx].length = NULL; + pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_FLOAT: - data->pBind[bindIdx].buffer_length = sizeof(float); - data->pBind[bindIdx].buffer = data->floatData + rowIdx; - data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + pBase[bindIdx].buffer_length = sizeof(float); + pBase[bindIdx].buffer = data->floatData + rowIdx; + pBase[bindIdx].length = NULL; + pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_DOUBLE: - data->pBind[bindIdx].buffer_length = sizeof(double); - data->pBind[bindIdx].buffer = data->doubleData + rowIdx; - data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + pBase[bindIdx].buffer_length = sizeof(double); + pBase[bindIdx].buffer = data->doubleData + rowIdx; + pBase[bindIdx].length = NULL; + pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_VARCHAR: - data->pBind[bindIdx].buffer_length = gVarCharSize; - data->pBind[bindIdx].buffer = data->binaryData + rowIdx * gVarCharSize; - data->pBind[bindIdx].length = data->binaryLen; - data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + pBase[bindIdx].buffer_length = gVarCharSize; + pBase[bindIdx].buffer = data->binaryData + rowIdx * gVarCharSize; + pBase[bindIdx].length = data->binaryLen; + pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_TIMESTAMP: - data->pBind[bindIdx].buffer_length = sizeof(int64_t); - data->pBind[bindIdx].buffer = data->tsData + rowIdx; - data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = NULL; + pBase[bindIdx].buffer_length = sizeof(int64_t); + pBase[bindIdx].buffer = data->tsData + rowIdx; + pBase[bindIdx].length = NULL; + pBase[bindIdx].is_null = NULL; break; case TSDB_DATA_TYPE_NCHAR: - data->pBind[bindIdx].buffer_length = gVarCharSize; - data->pBind[bindIdx].buffer = data->binaryData + rowIdx * gVarCharSize; - data->pBind[bindIdx].length = data->binaryLen; - data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + pBase[bindIdx].buffer_length = gVarCharSize; + pBase[bindIdx].buffer = data->binaryData + rowIdx * gVarCharSize; + pBase[bindIdx].length = data->binaryLen; + pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_UTINYINT: - data->pBind[bindIdx].buffer_length = sizeof(uint8_t); - data->pBind[bindIdx].buffer = data->utinyData + rowIdx; - data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + pBase[bindIdx].buffer_length = sizeof(uint8_t); + pBase[bindIdx].buffer = data->utinyData + rowIdx; + pBase[bindIdx].length = NULL; + pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_USMALLINT: - data->pBind[bindIdx].buffer_length = sizeof(uint16_t); - data->pBind[bindIdx].buffer = data->usmallData + rowIdx; - data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + pBase[bindIdx].buffer_length = sizeof(uint16_t); + pBase[bindIdx].buffer = data->usmallData + rowIdx; + pBase[bindIdx].length = NULL; + pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_UINT: - data->pBind[bindIdx].buffer_length = sizeof(uint32_t); - data->pBind[bindIdx].buffer = data->uintData + rowIdx; - data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + pBase[bindIdx].buffer_length = sizeof(uint32_t); + pBase[bindIdx].buffer = data->uintData + rowIdx; + pBase[bindIdx].length = NULL; + pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_UBIGINT: - data->pBind[bindIdx].buffer_length = sizeof(uint64_t); - data->pBind[bindIdx].buffer = data->ubigData + rowIdx; - data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + pBase[bindIdx].buffer_length = sizeof(uint64_t); + pBase[bindIdx].buffer = data->ubigData + rowIdx; + pBase[bindIdx].length = NULL; + pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; default: printf("!!!invalid col type:%d", dataType); exit(1); } - data->pBind[bindIdx].buffer_type = dataType; - data->pBind[bindIdx].num = gCurCase->bindRowNum; + pBase[bindIdx].buffer_type = dataType; + pBase[bindIdx].num = gCurCase->bindRowNum; + if (bType == BP_BIND_TAG) { + pBase[bindIdx].num = 1; + } + return 0; } @@ -694,6 +822,7 @@ int32_t prepareInsertData(BindData *data) { data->colTypes = taosMemoryCalloc(30, sizeof(int32_t)); data->sql = taosMemoryCalloc(1, 1024); data->pBind = taosMemoryCalloc((allRowNum/gCurCase->bindRowNum)*gCurCase->bindColNum, sizeof(TAOS_MULTI_BIND)); + data->pTags = taosMemoryCalloc(gCurCase->tblNum*gCurCase->bindTagNum, sizeof(TAOS_MULTI_BIND)); data->tsData = taosMemoryMalloc(allRowNum * sizeof(int64_t)); data->boolData = taosMemoryMalloc(allRowNum * sizeof(bool)); data->tinyData = taosMemoryMalloc(allRowNum * sizeof(int8_t)); @@ -734,10 +863,17 @@ int32_t prepareInsertData(BindData *data) { for (int b = 0; b < (allRowNum/gCurCase->bindRowNum); b++) { for (int c = 0; c < gCurCase->bindColNum; ++c) { - prepareColData(data, b*gCurCase->bindColNum+c, b*gCurCase->bindRowNum, c); + prepareColData(BP_BIND_COL, data, b*gCurCase->bindColNum+c, b*gCurCase->bindRowNum, c); } } + for (int b = 0; b < gCurCase->tblNum; b++) { + for (int c = 0; c < gCurCase->bindTagNum; ++c) { + prepareColData(BP_BIND_TAG, data, b*gCurCase->bindTagNum+c, b, c); + } + } + + generateInsertSQL(data); return 0; @@ -791,7 +927,7 @@ int32_t prepareQueryCondData(BindData *data, int32_t tblIdx) { for (int b = 0; b < bindNum; b++) { for (int c = 0; c < gCurCase->bindColNum; ++c) { - prepareColData(data, b*gCurCase->bindColNum+c, b*gCurCase->bindRowNum, c); + prepareColData(BP_BIND_COL, data, b*gCurCase->bindColNum+c, b*gCurCase->bindRowNum, c); } } @@ -849,7 +985,7 @@ int32_t prepareQueryMiscData(BindData *data, int32_t tblIdx) { for (int b = 0; b < bindNum; b++) { for (int c = 0; c < gCurCase->bindColNum; ++c) { - prepareColData(data, b*gCurCase->bindColNum+c, b*gCurCase->bindRowNum, c); + prepareColData(BP_BIND_COL, data, b*gCurCase->bindColNum+c, b*gCurCase->bindRowNum, c); } } @@ -1502,6 +1638,73 @@ int insertMPMETest1(TAOS_STMT *stmt, TAOS *taos) { return 0; } +/* [prepare [settbnametag [bind add] exec]] */ +int insertAUTOTest1(TAOS_STMT *stmt, TAOS *taos) { + int32_t loop = 0; + + while (gCurCase->bindColNum >= 2) { + BindData data = {0}; + prepareInsertData(&data); + + int code = taos_stmt_prepare(stmt, data.sql, 0); + if (code != 0){ + printf("!!!failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + bpCheckIsInsert(stmt, 1); + + int32_t bindTimes = gCurCase->rowNum/gCurCase->bindRowNum; + for (int32_t t = 0; t< gCurCase->tblNum; ++t) { + if (gCurCase->tblNum > 1) { + char buf[32]; + sprintf(buf, "t%d", t); + code = taos_stmt_set_tbname_tags(stmt, buf, data.pTags + t * gCurCase->bindTagNum); + if (code != 0){ + printf("!!!taos_stmt_set_tbname_tags error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } + + if (gCaseCtrl.checkParamNum) { + bpCheckParamNum(stmt); + } + + for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { + exit(1); + } + + if (taos_stmt_add_batch(stmt)) { + printf("!!!taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } + + if (taos_stmt_execute(stmt) != 0) { + printf("!!!taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } + + bpCheckIsInsert(stmt, 1); + + destroyData(&data); + + gCurCase->bindColNum -= 2; + gCurCase->bindTagNum -= 2; + gCurCase->fullCol = false; + loop++; + } + + bpCheckAffectedRows(stmt, loop); + + gExecLoopTimes = loop; + + return 0; +} + + /* select * from table */ int querySUBTTest1(TAOS_STMT *stmt, TAOS *taos) { BindData data = {0}; @@ -2911,63 +3114,6 @@ int sql_s_perf1(TAOS *taos) { void generateCreateTableSQL(char *buf, int32_t tblIdx, int32_t colNum, int32_t *colList, bool stable) { int32_t blen = 0; blen = sprintf(buf, "create table %s%d ", (stable ? bpStbPrefix : bpTbPrefix), tblIdx); - if (stable) { - blen += sprintf(buf + blen, "tags ("); - for (int c = 0; c < colNum; ++c) { - if (c > 0) { - blen += sprintf(buf + blen, ","); - } - switch (colList[c]) { - case TSDB_DATA_TYPE_BOOL: - blen += sprintf(buf + blen, "tbooldata bool"); - break; - case TSDB_DATA_TYPE_TINYINT: - blen += sprintf(buf + blen, "ttinydata tinyint"); - break; - case TSDB_DATA_TYPE_SMALLINT: - blen += sprintf(buf + blen, "tsmalldata smallint"); - break; - case TSDB_DATA_TYPE_INT: - blen += sprintf(buf + blen, "tintdata int"); - break; - case TSDB_DATA_TYPE_BIGINT: - blen += sprintf(buf + blen, "tbigdata bigint"); - break; - case TSDB_DATA_TYPE_FLOAT: - blen += sprintf(buf + blen, "tfloatdata float"); - break; - case TSDB_DATA_TYPE_DOUBLE: - blen += sprintf(buf + blen, "tdoubledata double"); - break; - case TSDB_DATA_TYPE_VARCHAR: - blen += sprintf(buf + blen, "tbinarydata binary(%d)", gVarCharSize); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - blen += sprintf(buf + blen, "tts ts"); - break; - case TSDB_DATA_TYPE_NCHAR: - blen += sprintf(buf + blen, "tnchardata nchar(%d)", gVarCharSize); - break; - case TSDB_DATA_TYPE_UTINYINT: - blen += sprintf(buf + blen, "tutinydata tinyint unsigned"); - break; - case TSDB_DATA_TYPE_USMALLINT: - blen += sprintf(buf + blen, "tusmalldata smallint unsigned"); - break; - case TSDB_DATA_TYPE_UINT: - blen += sprintf(buf + blen, "tuintdata int unsigned"); - break; - case TSDB_DATA_TYPE_UBIGINT: - blen += sprintf(buf + blen, "tubigdata bigint unsigned"); - break; - default: - printf("invalid col type:%d", colList[c]); - exit(1); - } - } - - blen += sprintf(buf + blen, ")"); - } blen += sprintf(buf + blen, " ("); @@ -3027,6 +3173,64 @@ void generateCreateTableSQL(char *buf, int32_t tblIdx, int32_t colNum, int32_t * blen += sprintf(buf + blen, ")"); + if (stable) { + blen += sprintf(buf + blen, "tags ("); + for (int c = 0; c < colNum; ++c) { + if (c > 0) { + blen += sprintf(buf + blen, ","); + } + switch (colList[c]) { + case TSDB_DATA_TYPE_BOOL: + blen += sprintf(buf + blen, "tbooldata bool"); + break; + case TSDB_DATA_TYPE_TINYINT: + blen += sprintf(buf + blen, "ttinydata tinyint"); + break; + case TSDB_DATA_TYPE_SMALLINT: + blen += sprintf(buf + blen, "tsmalldata smallint"); + break; + case TSDB_DATA_TYPE_INT: + blen += sprintf(buf + blen, "tintdata int"); + break; + case TSDB_DATA_TYPE_BIGINT: + blen += sprintf(buf + blen, "tbigdata bigint"); + break; + case TSDB_DATA_TYPE_FLOAT: + blen += sprintf(buf + blen, "tfloatdata float"); + break; + case TSDB_DATA_TYPE_DOUBLE: + blen += sprintf(buf + blen, "tdoubledata double"); + break; + case TSDB_DATA_TYPE_VARCHAR: + blen += sprintf(buf + blen, "tbinarydata binary(%d)", gVarCharSize); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + blen += sprintf(buf + blen, "tts timestamp"); + break; + case TSDB_DATA_TYPE_NCHAR: + blen += sprintf(buf + blen, "tnchardata nchar(%d)", gVarCharSize); + break; + case TSDB_DATA_TYPE_UTINYINT: + blen += sprintf(buf + blen, "tutinydata tinyint unsigned"); + break; + case TSDB_DATA_TYPE_USMALLINT: + blen += sprintf(buf + blen, "tusmalldata smallint unsigned"); + break; + case TSDB_DATA_TYPE_UINT: + blen += sprintf(buf + blen, "tuintdata int unsigned"); + break; + case TSDB_DATA_TYPE_UBIGINT: + blen += sprintf(buf + blen, "tubigdata bigint unsigned"); + break; + default: + printf("invalid col type:%d", colList[c]); + exit(1); + } + } + + blen += sprintf(buf + blen, ")"); + } + if (gCaseCtrl.printCreateTblSql) { printf("\tCreate Table SQL:%s\n", buf); } @@ -3067,7 +3271,7 @@ void prepare(TAOS *taos, int32_t colNum, int32_t *colList, int prepareStb) { } } else { char buf[1024]; - generateCreateTableSQL(buf, 1, colNum, colList, true); + generateCreateTableSQL(buf, bpDefaultStbId, colNum, colList, true); result = taos_query(taos, buf); code = taos_errno(result); @@ -3113,14 +3317,18 @@ int32_t runCase(TAOS *taos, int32_t caseIdx, int32_t caseRunIdx, bool silent) { if (gCurCase->fullCol) { gCurCase->bindColNum = gCurCase->colNum; + gCurCase->bindTagNum = gCurCase->colNum; } gCurCase->bindNullNum = gCaseCtrl.bindNullNum; - gCurCase->prepareStb = gCaseCtrl.prepareStb; if (gCaseCtrl.bindColNum) { gCurCase->bindColNum = gCaseCtrl.bindColNum; gCurCase->fullCol = false; } + if (gCaseCtrl.bindTagNum) { + gCurCase->bindTagNum = gCaseCtrl.bindTagNum; + gCurCase->fullCol = false; + } if (gCaseCtrl.bindRowNum) { gCurCase->bindRowNum = gCaseCtrl.bindRowNum; } @@ -3128,6 +3336,10 @@ int32_t runCase(TAOS *taos, int32_t caseIdx, int32_t caseRunIdx, bool silent) { gCurCase->bindColNum = gCaseCtrl.bindColTypeNum; gCurCase->fullCol = false; } + if (gCaseCtrl.bindTagTypeNum) { + gCurCase->bindTagNum = gCaseCtrl.bindTagTypeNum; + gCurCase->fullCol = false; + } if (!silent) { printf("* Case %d - [%s]%s Begin *\n", caseRunIdx, gCaseCtrl.caseCatalog, gCurCase->caseDesc); From 1f4749439f3f0d5c6bad33db1401c3613805d83e Mon Sep 17 00:00:00 2001 From: dapan Date: Mon, 9 May 2022 07:59:07 +0800 Subject: [PATCH 03/29] stmt test --- tests/script/api/batchprepare.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 3dddb03f3b..6afe0a5cd1 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -244,6 +244,7 @@ CaseCtrl gCaseCtrl = { // query case with specified col&oper .printStmtSql = true, .rowNum = 0, .bindColNum = 0, + .bindTagNum = 0, .bindRowNum = 0, .bindColTypeNum = 0, .bindColTypeList = NULL, From 6621ee69a049a249c15c6bec2152145936a14530 Mon Sep 17 00:00:00 2001 From: dapan Date: Mon, 9 May 2022 11:40:24 +0800 Subject: [PATCH 04/29] fix bug --- source/common/src/tmsg.c | 6 +++--- source/libs/executor/src/executorimpl.c | 6 ++++-- source/libs/parser/src/parTranslater.c | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 56bb93faa4..85b55611dc 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1233,21 +1233,21 @@ int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRs for (int32_t i = 0; i < numOfCreatedDbs; ++i) { char db[TSDB_DB_FNAME_LEN] = {0}; if (tDecodeCStrTo(pDecoder, db) < 0) return -1; - int32_t len = strlen(db) + 1; + int32_t len = strlen(db); taosHashPut(pRsp->createdDbs, db, len, db, len); } for (int32_t i = 0; i < numOfReadDbs; ++i) { char db[TSDB_DB_FNAME_LEN] = {0}; if (tDecodeCStrTo(pDecoder, db) < 0) return -1; - int32_t len = strlen(db) + 1; + int32_t len = strlen(db); taosHashPut(pRsp->readDbs, db, len, db, len); } for (int32_t i = 0; i < numOfWriteDbs; ++i) { char db[TSDB_DB_FNAME_LEN] = {0}; if (tDecodeCStrTo(pDecoder, db) < 0) return -1; - int32_t len = strlen(db) + 1; + int32_t len = strlen(db); taosHashPut(pRsp->writeDbs, db, len, db, len); } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index e4437f1a93..45995173e6 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4206,7 +4206,7 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) { } } - taosMemoryFree(pOperator->pExpr); + taosMemoryFreeClear(pOperator->pExpr); taosMemoryFreeClear(pOperator->info); taosMemoryFreeClear(pOperator); } @@ -4391,6 +4391,9 @@ void destroySFillOperatorInfo(void* param, int32_t numOfOutput) { } static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) { + if (NULL == param) { + return; + } SProjectOperatorInfo* pInfo = (SProjectOperatorInfo*)param; doDestroyBasicInfo(&pInfo->binfo, numOfOutput); cleanupAggSup(&pInfo->aggSup); @@ -5328,7 +5331,6 @@ void doDestroyTask(SExecTaskInfo* pTaskInfo) { // taosArrayDestroy(pTaskInfo->summary.queryProfEvents); // taosHashCleanup(pTaskInfo->summary.operatorProfResults); - destroyOperatorInfo(pTaskInfo->pRoot); taosMemoryFreeClear(pTaskInfo->sql); taosMemoryFreeClear(pTaskInfo->id.str); taosMemoryFreeClear(pTaskInfo); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 63ef3fc9a1..c8a3a89e40 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3146,7 +3146,7 @@ static int32_t translateGrant(STranslateContext* pCxt, SGrantStmt* pStmt) { req.alterType = TSDB_ALTER_USER_ADD_WRITE_DB; } strcpy(req.user, pStmt->userName); - strcpy(req.dbname, pStmt->dbName); + sprintf(req.dbname, "%d.%s", pCxt->pParseCxt->acctId, pStmt->dbName); return buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &req); } From b6593d9ae9b9a6ba7f8925f56991967fa66a9422 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 9 May 2022 05:06:55 +0000 Subject: [PATCH 05/29] refact: tsdb --- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 139 ++++++++++++++------ 1 file changed, 100 insertions(+), 39 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index 952ccfda9c..e162d519ac 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -22,15 +22,16 @@ typedef struct SMemSkipListNode SMemSkipListNode; typedef struct SMemSkipListCurosr SMemSkipListCurosr; struct SMemTable { - STsdb *pTsdb; - TSKEY minKey; - TSKEY maxKey; - int64_t minVer; - int64_t maxVer; - int64_t nRows; - int32_t nHash; - int32_t nBucket; - SMemData **pBuckets; + STsdb *pTsdb; + TSKEY minKey; + TSKEY maxKey; + int64_t minVer; + int64_t maxVer; + int64_t nRows; + int32_t nHash; + int32_t nBucket; + SMemData **pBuckets; + SMemSkipListCurosr *pSlc; }; struct SMemSkipListNode { @@ -60,9 +61,15 @@ struct SMemData { struct SMemSkipListCurosr { SMemSkipList *pSl; - SMemSkipListNode *pNodeC; + SMemSkipListNode *forwards[]; }; +typedef struct { + int64_t version; + uint32_t szRow; + const STSRow *pRow; +} STsdbRow; + #define HASH_BUCKET(SUID, UID, NBUCKET) (TABS((SUID) + (UID)) % (NBUCKET)) #define SL_NODE_SIZE(l) (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l)*2) @@ -76,7 +83,13 @@ struct SMemSkipListCurosr { #define SL_HEAD_NODE_FORWARD(n, l) SL_NODE_FORWARD(n, l) #define SL_TAIL_NODE_BACKWARD(n, l) SL_NODE_FORWARD(n, l) -static int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl); +static int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl); +static int32_t tsdbEncodeRow(SEncoder *pEncoder, const STsdbRow *pRow); +static int32_t tsdbDecodeRow(SDecoder *pDecoder, STsdbRow *pRow); +static int32_t tsdbMemSkipListCursorCreate(int8_t maxLevel, SMemSkipListCurosr **ppSlc); +static void tsdbMemSkipListCursorDestroy(SMemSkipListCurosr *pSlc); +static void tsdbMemSkipListCursorInit(SMemSkipListCurosr *pSlc, SMemSkipList *pSl); +static void tsdbMemSkipListCursorPut(SMemSkipListCurosr *pSlc, SMemSkipListNode *pNode); // SMemTable int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) { @@ -102,6 +115,11 @@ int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) { taosMemoryFree(pMemTb); return -1; } + if (tsdbMemSkipListCursorCreate(pTsdb->pVnode->config.tsdbCfg.slLevel, &pMemTb->pSlc) < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pMemTb->pBuckets); + taosMemoryFree(pMemTb); + } *ppMemTb = pMemTb; return 0; @@ -110,6 +128,7 @@ int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) { int32_t tsdbMemTableDestroy2(STsdb *pTsdb, SMemTable *pMemTb) { if (pMemTb) { // loop to destroy the contents (todo) + tsdbMemSkipListCursorDestroy(pMemTb->pSlc); taosMemoryFree(pMemTb->pBuckets); taosMemoryFree(pMemTb); } @@ -177,52 +196,47 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p } // do insert data to SMemData - SMemSkipListCurosr slc = {0}; - const STSRow *pRow; - uint32_t szRow; - SDecoder decoder = {0}; + STsdbRow tRow = {.version = version}; + SEncoder ec = {0}; + SDecoder dc = {0}; - tDecoderInit(&decoder, pSubmitBlk->pData, pSubmitBlk->nData); + tDecoderInit(&dc, pSubmitBlk->pData, pSubmitBlk->nData); + tsdbMemSkipListCursorInit(pMemTb->pSlc, &pMemData->sl); for (;;) { - if (tDecodeIsEnd(&decoder)) break; + if (tDecodeIsEnd(&dc)) break; - if (tDecodeBinary(&decoder, (const uint8_t **)&pRow, &szRow) < 0) { + if (tDecodeBinary(&dc, (const uint8_t **)&tRow.pRow, &tRow.szRow) < 0) { terrno = TSDB_CODE_INVALID_MSG; return -1; } - // check the row (todo) - - // // move the cursor to position to write (todo) - // int32_t c; - // tsdbMemSkipListCursorMoveTo(&slc, pTSRow, version, &c); - // ASSERT(c); + // move cursor // encode row - int8_t level = tsdbMemSkipListRandLevel(&pMemData->sl); - int32_t tsize = SL_NODE_SIZE(level) + sizeof(version) + (0 /*todo*/); - SMemSkipListNode *pNode = vnodeBufPoolMalloc(pPool, tsize); + int8_t level = tsdbMemSkipListRandLevel(&pMemData->sl); + int32_t tsize; + int32_t ret; + tEncodeSize(tsdbEncodeRow, &tRow, tsize, ret); + SMemSkipListNode *pNode = vnodeBufPoolMalloc(pPool, tsize + SL_NODE_SIZE(level)); if (pNode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } pNode->level = level; + tEncoderInit(&ec, (uint8_t *)SL_NODE_DATA(pNode), tsize); + ret = tsdbEncodeRow(&ec, &tRow); + ASSERT(ret == 0); + tEncoderClear(&ec); - // uint8_t *pData = SL_NODE_DATA(pSlNode); - // *(int64_t *)pData = version; - // pData += sizeof(version); - // memcpy(pData, pt, p - pt); - - // // insert row - // tsdbMemSkipListCursorPut(&slc, pSlNode); + // put the node + tsdbMemSkipListCursorPut(pMemTb->pSlc, pNode); // update status - if (pRow->ts < pMemData->minKey) pMemData->minKey = pRow->ts; - if (pRow->ts > pMemData->maxKey) pMemData->maxKey = pRow->ts; + if (tRow.pRow->ts < pMemData->minKey) pMemData->minKey = tRow.pRow->ts; + if (tRow.pRow->ts > pMemData->maxKey) pMemData->maxKey = tRow.pRow->ts; } - tDecoderClear(&decoder); - // tsdbMemSkipListCursorClose(&slc); + tDecoderClear(&dc); // update status if (pMemData->minVer == -1) pMemData->minVer = version; @@ -236,7 +250,7 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p return 0; } -static int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl) { +static FORCE_INLINE int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl) { int8_t level = 1; int8_t tlevel; const uint32_t factor = 4; @@ -249,4 +263,51 @@ static int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl) { } return level; +} + +static FORCE_INLINE int32_t tsdbEncodeRow(SEncoder *pEncoder, const STsdbRow *pRow) { + if (tEncodeI64(pEncoder, pRow->version) < 0) return -1; + if (tEncodeBinary(pEncoder, (const uint8_t *)pRow->pRow, pRow->szRow) < 0) return -1; + return 0; +} + +static FORCE_INLINE int32_t tsdbDecodeRow(SDecoder *pDecoder, STsdbRow *pRow) { + if (tDecodeI64(pDecoder, &pRow->version) < 0) return -1; + if (tDecodeBinary(pDecoder, (const uint8_t **)&pRow->pRow, &pRow->szRow) < 0) return -1; + return 0; +} + +static int32_t tsdbMemSkipListCursorCreate(int8_t maxLevel, SMemSkipListCurosr **ppSlc) { + *ppSlc = (SMemSkipListCurosr *)taosMemoryCalloc(1, sizeof(**ppSlc) + sizeof(SMemSkipListNode *) * maxLevel); + if (*ppSlc == NULL) { + return -1; + } + return 0; +} + +static void tsdbMemSkipListCursorDestroy(SMemSkipListCurosr *pSlc) { taosMemoryFree(pSlc); } + +static void tsdbMemSkipListCursorInit(SMemSkipListCurosr *pSlc, SMemSkipList *pSl) { + SMemSkipListNode *pHead = SL_HEAD_NODE(pSl); + pSlc->pSl = pSl; + for (int8_t iLevel = 0; iLevel < pSl->maxLevel; iLevel++) { + pSlc->forwards[iLevel] = pHead; + } +} + +static void tsdbMemSkipListCursorPut(SMemSkipListCurosr *pSlc, SMemSkipListNode *pNode) { + SMemSkipList *pSl = pSlc->pSl; + SMemSkipListNode *pNodeNext; + + for (int8_t iLevel = 0; iLevel < pNode->level; iLevel++) { + // todo + + ASSERT(0); + } + + if (pSl->level < pNode->level) { + pSl->level = pNode->level; + } + + pSl->size += 1; } \ No newline at end of file From 9a035fdb1dbab8257c769097006df8b679bd05f2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 9 May 2022 05:40:25 +0000 Subject: [PATCH 06/29] refact tsdb --- include/util/tskiplist2.h | 70 -------- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 42 ++++- source/util/src/tskiplist2.c | 175 -------------------- 3 files changed, 38 insertions(+), 249 deletions(-) delete mode 100644 include/util/tskiplist2.h delete mode 100644 source/util/src/tskiplist2.c diff --git a/include/util/tskiplist2.h b/include/util/tskiplist2.h deleted file mode 100644 index 83e6ad0868..0000000000 --- a/include/util/tskiplist2.h +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#ifndef _TD_UTIL_SKIPLIST2_H_ -#define _TD_UTIL_SKIPLIST2_H_ - -#include "os.h" - -#ifdef __cplusplus -extern "C" { -#endif - -#define SL_MAX_LEVEL 15 - -typedef struct SSkipList2 SSkipList2; -typedef struct SSLCursor SSLCursor; -typedef struct SSLCfg SSLCfg; -typedef struct SSLNode SSLNode; - -typedef int32_t (*tslCmprFn)(const void *pKey1, int32_t nKey1, const void *pKey2, int32_t nKey2); - -// SSkipList2 -int32_t slOpen(const SSLCfg *pCfg, SSkipList2 **ppSl); -int32_t slClose(SSkipList2 *pSl); -int32_t slClear(SSkipList2 *pSl); - -// SSLCursor -int32_t slcOpen(SSkipList2 *pSl, SSLCursor *pSlc); -int32_t slcClose(SSLCursor *pSlc); -int32_t slcMoveTo(SSLCursor *pSlc, const void *pKey, int32_t nKey); -int32_t slcMoveToNext(SSLCursor *pSlc); -int32_t slcMoveToPrev(SSLCursor *pSlc); -int32_t slcMoveToFirst(SSLCursor *pSlc); -int32_t slcMoveToLast(SSLCursor *pSlc); -int32_t slcPut(SSLCursor *pSlc, const void *pKey, int32_t nKey, const void *pData, int32_t nData); -int32_t slcGet(SSLCursor *pSlc, const void **ppKey, int32_t *nKey, const void **ppData, int32_t *nData); -int32_t slcDrop(SSLCursor *pSlc); - -// struct -struct SSLCfg { - int8_t maxLevel; - int32_t nKey; - int32_t nData; - tslCmprFn cmprFn; - void *pPool; - void *(*xMalloc)(void *, int32_t size); - void (*xFree)(void *, void *); -}; - -struct SSLCursor { - SSkipList2 *pSl; - SSLNode **forwards[SL_MAX_LEVEL]; -}; - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_UTIL_SKIPLIST2_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index e162d519ac..e8e7260514 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -61,7 +61,7 @@ struct SMemData { struct SMemSkipListCurosr { SMemSkipList *pSl; - SMemSkipListNode *forwards[]; + SMemSkipListNode *pNodeC; }; typedef struct { @@ -70,6 +70,8 @@ typedef struct { const STSRow *pRow; } STsdbRow; +#define SL_MAX_LEVEL 15 + #define HASH_BUCKET(SUID, UID, NBUCKET) (TABS((SUID) + (UID)) % (NBUCKET)) #define SL_NODE_SIZE(l) (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l)*2) @@ -90,6 +92,9 @@ static int32_t tsdbMemSkipListCursorCreate(int8_t maxLevel, SMemSkipListCurosr * static void tsdbMemSkipListCursorDestroy(SMemSkipListCurosr *pSlc); static void tsdbMemSkipListCursorInit(SMemSkipListCurosr *pSlc, SMemSkipList *pSl); static void tsdbMemSkipListCursorPut(SMemSkipListCurosr *pSlc, SMemSkipListNode *pNode); +static int32_t tsdbMemSkipListCursorMoveTo(SMemSkipListCurosr *pSlc, int64_t version, TSKEY ts, int32_t flags); +static int32_t tsdbMemSkipListCursorMoveToNext(SMemSkipListCurosr *pSlc); +static int32_t tsdbMemSkipListCursorMoveToPrev(SMemSkipListCurosr *pSlc); // SMemTable int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) { @@ -211,6 +216,7 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p } // move cursor + tsdbMemSkipListCursorMoveTo(pMemTb->pSlc, version, tRow.pRow->ts, 0); // encode row int8_t level = tsdbMemSkipListRandLevel(&pMemData->sl); @@ -290,9 +296,9 @@ static void tsdbMemSkipListCursorDestroy(SMemSkipListCurosr *pSlc) { taosMemoryF static void tsdbMemSkipListCursorInit(SMemSkipListCurosr *pSlc, SMemSkipList *pSl) { SMemSkipListNode *pHead = SL_HEAD_NODE(pSl); pSlc->pSl = pSl; - for (int8_t iLevel = 0; iLevel < pSl->maxLevel; iLevel++) { - pSlc->forwards[iLevel] = pHead; - } + // for (int8_t iLevel = 0; iLevel < pSl->maxLevel; iLevel++) { + // pSlc->forwards[iLevel] = pHead; + // } } static void tsdbMemSkipListCursorPut(SMemSkipListCurosr *pSlc, SMemSkipListNode *pNode) { @@ -310,4 +316,32 @@ static void tsdbMemSkipListCursorPut(SMemSkipListCurosr *pSlc, SMemSkipListNode } pSl->size += 1; +} + +static int32_t tsdbMemSkipListCursorMoveTo(SMemSkipListCurosr *pSlc, int64_t version, TSKEY ts, int32_t flags) { + SMemSkipListNode **pForwards = NULL; + SMemSkipList *pSl = pSlc->pSl; + int8_t maxLevel = pSl->maxLevel; + SMemSkipListNode *pHead = SL_HEAD_NODE(pSl); + SMemSkipListNode *pTail = SL_TAIL_NODE(pSl); + + for (int8_t iLevel = 0; iLevel < maxLevel; iLevel++) { + pForwards[iLevel] = pHead; + } + + for (int8_t iLevel = maxLevel - 1; iLevel >= 0; iLevel--) { + if (iLevel < pSl->level) { + } + } + return 0; +} + +static int32_t tsdbMemSkipListCursorMoveToNext(SMemSkipListCurosr *pSlc) { + // TODO + return 0; +} + +static int32_t tsdbMemSkipListCursorMoveToPrev(SMemSkipListCurosr *pSlc) { + // TODO + return 0; } \ No newline at end of file diff --git a/source/util/src/tskiplist2.c b/source/util/src/tskiplist2.c deleted file mode 100644 index 77f5ed5051..0000000000 --- a/source/util/src/tskiplist2.c +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tskiplist2.h" - -struct SSLNode { - int8_t level; - SSLNode *forwards[]; -}; - -struct SSkipList2 { - int8_t level; - uint32_t seed; - int32_t size; - const SSLCfg *pCfg; - SSLNode *pHead[]; -}; - -static void *slMalloc(void *pPool, int32_t size); -static void slFree(void *pPool, void *p); -static int32_t slCmprFn(const void *pKey, int32_t nKey, const void *pData, int32_t nData); - -const SSLCfg slDefaultCfg = {.maxLevel = SL_MAX_LEVEL, - .nKey = -1, - .nData = -1, - .cmprFn = slCmprFn, - .pPool = NULL, - .xMalloc = slMalloc, - .xFree = slFree}; - -int32_t slOpen(const SSLCfg *pCfg, SSkipList2 **ppSl) { - SSkipList2 *pSl = NULL; - int32_t size; - - *ppSl = NULL; - if (pCfg == NULL) pCfg = &slDefaultCfg; - - // check config (TODO) - - // malloc handle - size = sizeof(*pSl) + sizeof(SSLNode *) * pCfg->maxLevel * 2; - pSl = pCfg->xMalloc(pCfg->pPool, size); - if (pSl == NULL) { - return -1; - } - - pSl->level = 0; - pSl->seed = taosRand(); - pSl->size = 0; - pSl->pCfg = pCfg; - - // init an empty skiplist - for (int32_t i = 0; i < pCfg->maxLevel * 2; i++) { - pSl->pHead[i] = NULL; - } - - *ppSl = pSl; - return 0; -} - -int32_t slClose(SSkipList2 *pSl) { - if (pSl) { - slClear(pSl); - if (pSl->pCfg->xFree) { - pSl->pCfg->xFree(pSl->pCfg->pPool, pSl); - } - } - - return 0; -} - -int32_t slClear(SSkipList2 *pSl) { - // loop to clear sl - for (;;) { - // (TODO) - } - - // init sl (TODO) - - return 0; -} - -int32_t slcOpen(SSkipList2 *pSl, SSLCursor *pSlc) { - pSlc->pSl = pSl; - - for (int i = 0; i < SL_MAX_LEVEL; i++) { - if (i < pSl->pCfg->maxLevel) { - } else { - pSlc->forwards[i] = NULL; - } - } - - // TODO - return 0; -} - -int32_t slcClose(SSLCursor *pSlc) { - // TODO - return 0; -} - -int32_t slcMoveTo(SSLCursor *pSlc, const void *pKey, int32_t nKey) { - // TODO - return 0; -} - -int32_t slcMoveToNext(SSLCursor *pSlc) { - // TODO - return 0; -} - -int32_t slcMoveToPrev(SSLCursor *pSlc) { - // TODO - return 0; -} - -int32_t slcMoveToFirst(SSLCursor *pSlc) { - // TODO - return 0; -} - -int32_t slcMoveToLast(SSLCursor *pSlc) { - // TODO - return 0; -} - -int32_t slcPut(SSLCursor *pSlc, const void *pKey, int32_t nKey, const void *pData, int32_t nData) { - // TODO - return 0; -} - -int32_t slcGet(SSLCursor *pSlc, const void **ppKey, int32_t *nKey, const void **ppData, int32_t *nData) { - // TODO - return 0; -} - -int32_t slcDrop(SSLCursor *pSlc) { - // TODO - return 0; -} - -static FORCE_INLINE void *slMalloc(void *pPool, int32_t size) { return taosMemoryMalloc(size); } - -static FORCE_INLINE void slFree(void *pPool, void *p) { taosMemoryFree(p); } - -static int32_t slCmprFn(const void *pKey1, int32_t nKey1, const void *pKey2, int32_t nKey2) { - ASSERT(nKey1 >= 0 && nKey2 >= 0); - - int32_t nKey = nKey1 > nKey2 ? nKey2 : nKey1; - int32_t c; - - c = memcmp(pKey1, pKey2, nKey); - if (c == 0) { - if (nKey1 > nKey2) { - c = 1; - } else if (nKey1 < nKey2) { - c = -1; - } - } - - return c; -} \ No newline at end of file From fba58c763f573a03242efbf4994bf98cc2526a2f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 9 May 2022 06:29:43 +0000 Subject: [PATCH 07/29] refact TSDB --- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 60 ++++++++++++--------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index e8e7260514..bfd79f7965 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -95,6 +95,7 @@ static void tsdbMemSkipListCursorPut(SMemSkipListCurosr *pSlc, SMemSkipListNo static int32_t tsdbMemSkipListCursorMoveTo(SMemSkipListCurosr *pSlc, int64_t version, TSKEY ts, int32_t flags); static int32_t tsdbMemSkipListCursorMoveToNext(SMemSkipListCurosr *pSlc); static int32_t tsdbMemSkipListCursorMoveToPrev(SMemSkipListCurosr *pSlc); +static SMemSkipListNode *tsdbMemSkipListNodeCreate(SVBufPool *pPool, SMemSkipList *pSl, const STsdbRow *pTRow); // SMemTable int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) { @@ -201,15 +202,18 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p } // do insert data to SMemData - STsdbRow tRow = {.version = version}; - SEncoder ec = {0}; - SDecoder dc = {0}; + SMemSkipListNode *forwards[SL_MAX_LEVEL]; + SMemSkipListNode *pNode; + STsdbRow tRow = {.version = version}; + SEncoder ec = {0}; + SDecoder dc = {0}; tDecoderInit(&dc, pSubmitBlk->pData, pSubmitBlk->nData); tsdbMemSkipListCursorInit(pMemTb->pSlc, &pMemData->sl); for (;;) { if (tDecodeIsEnd(&dc)) break; + // decode row if (tDecodeBinary(&dc, (const uint8_t **)&tRow.pRow, &tRow.szRow) < 0) { terrno = TSDB_CODE_INVALID_MSG; return -1; @@ -219,22 +223,12 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p tsdbMemSkipListCursorMoveTo(pMemTb->pSlc, version, tRow.pRow->ts, 0); // encode row - int8_t level = tsdbMemSkipListRandLevel(&pMemData->sl); - int32_t tsize; - int32_t ret; - tEncodeSize(tsdbEncodeRow, &tRow, tsize, ret); - SMemSkipListNode *pNode = vnodeBufPoolMalloc(pPool, tsize + SL_NODE_SIZE(level)); + pNode = tsdbMemSkipListNodeCreate(pPool, &pMemData->sl, &tRow); if (pNode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pNode->level = level; - tEncoderInit(&ec, (uint8_t *)SL_NODE_DATA(pNode), tsize); - ret = tsdbEncodeRow(&ec, &tRow); - ASSERT(ret == 0); - tEncoderClear(&ec); - // put the node tsdbMemSkipListCursorPut(pMemTb->pSlc, pNode); @@ -258,14 +252,11 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p static FORCE_INLINE int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl) { int8_t level = 1; - int8_t tlevel; + int8_t tlevel = TMIN(pSl->maxLevel, pSl->level + 1); const uint32_t factor = 4; - if (pSl->size) { - tlevel = TMIN(pSl->maxLevel, pSl->level + 1); - while ((taosRandR(&pSl->seed) % factor) == 0 && level < tlevel) { - level++; - } + while ((taosRandR(&pSl->seed) % factor) == 0 && level < tlevel) { + level++; } return level; @@ -325,14 +316,12 @@ static int32_t tsdbMemSkipListCursorMoveTo(SMemSkipListCurosr *pSlc, int64_t ver SMemSkipListNode *pHead = SL_HEAD_NODE(pSl); SMemSkipListNode *pTail = SL_TAIL_NODE(pSl); - for (int8_t iLevel = 0; iLevel < maxLevel; iLevel++) { - pForwards[iLevel] = pHead; - } - - for (int8_t iLevel = maxLevel - 1; iLevel >= 0; iLevel--) { - if (iLevel < pSl->level) { + if (pSl->size == 0) { + for (int8_t iLevel = 0; iLevel < pSl->maxLevel; iLevel++) { + pForwards[iLevel] = pHead; } } + return 0; } @@ -344,4 +333,23 @@ static int32_t tsdbMemSkipListCursorMoveToNext(SMemSkipListCurosr *pSlc) { static int32_t tsdbMemSkipListCursorMoveToPrev(SMemSkipListCurosr *pSlc) { // TODO return 0; +} + +static SMemSkipListNode *tsdbMemSkipListNodeCreate(SVBufPool *pPool, SMemSkipList *pSl, const STsdbRow *pTRow) { + int32_t tsize; + int32_t ret; + int8_t level = tsdbMemSkipListRandLevel(pSl); + SMemSkipListNode *pNode = NULL; + SEncoder ec = {0}; + + tEncodeSize(tsdbEncodeRow, pTRow, tsize, ret); + pNode = vnodeBufPoolMalloc(pPool, tsize + SL_NODE_SIZE(level)); + if (pNode) { + pNode->level = level; + tEncoderInit(&ec, (uint8_t *)SL_NODE_DATA(pNode), tsize); + tsdbEncodeRow(&ec, pTRow); + tEncoderClear(&ec); + } + + return pNode; } \ No newline at end of file From ede0d49163a8d06722381627ae6bfe7479bac5f4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 9 May 2022 06:30:45 +0000 Subject: [PATCH 08/29] more refact --- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index bfd79f7965..2dff0f14e1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -204,13 +204,14 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p // do insert data to SMemData SMemSkipListNode *forwards[SL_MAX_LEVEL]; SMemSkipListNode *pNode; + int32_t iRow; STsdbRow tRow = {.version = version}; SEncoder ec = {0}; SDecoder dc = {0}; tDecoderInit(&dc, pSubmitBlk->pData, pSubmitBlk->nData); tsdbMemSkipListCursorInit(pMemTb->pSlc, &pMemData->sl); - for (;;) { + for (iRow = 0;; iRow++) { if (tDecodeIsEnd(&dc)) break; // decode row From bcfaa9f3353e35c76fe797dc8c153becade4bfe8 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Mon, 9 May 2022 14:59:23 +0800 Subject: [PATCH 09/29] add testcase --- .../1-insert/insertWithMoreVgroup.py | 6 +- tests/system-test/1-insert/manyVgroups.json | 2 +- tests/system-test/insert.json | 76 +++++++++++++++++++ 3 files changed, 81 insertions(+), 3 deletions(-) create mode 100644 tests/system-test/insert.json diff --git a/tests/system-test/1-insert/insertWithMoreVgroup.py b/tests/system-test/1-insert/insertWithMoreVgroup.py index b583ee93e8..ae450ae69a 100644 --- a/tests/system-test/1-insert/insertWithMoreVgroup.py +++ b/tests/system-test/1-insert/insertWithMoreVgroup.py @@ -219,7 +219,7 @@ class TDTestCase: for i in range(threadNumbers): jsonfile="1-insert/Vgroups%d%d.json"%(vgroups,i) os.system("cp -f 1-insert/manyVgroups.json %s"%(jsonfile)) - os.system("sed -i 's/\"name\": \"db\",/\"name\": \"%s%d\",/g' %s"%(dbname,i,jsonfile)) + os.system("sed -i 's/\"name\": \"db\",/\"name\": \"%s\",/g' %s"%(dbname,jsonfile)) os.system("sed -i 's/\"childtable_count\": 300000,/\"childtable_count\": %d,/g' %s "%(count,jsonfile)) os.system("sed -i 's/\"name\": \"stb1\",/\"name\": \"%s%d\",/g' %s "%(stbname,i,jsonfile)) os.system("sed -i 's/\"childtable_prefix\": \"stb1_\",/\"childtable_prefix\": \"%s%d_\",/g' %s "%(stbname,i,jsonfile)) @@ -337,9 +337,11 @@ class TDTestCase: return def test_case3(self): - self.taosBenchCreate("db1", "stb1", 1, 2, 1*50000) + # self.taosBenchCreate("db1", "stb1", 4, 5, 100*10000) + # self.taosBenchCreate("db1", "stb1", 1, 5, 100*10000) + return # diff --git a/tests/system-test/1-insert/manyVgroups.json b/tests/system-test/1-insert/manyVgroups.json index df6f1163e8..6ce78af334 100644 --- a/tests/system-test/1-insert/manyVgroups.json +++ b/tests/system-test/1-insert/manyVgroups.json @@ -16,7 +16,7 @@ { "dbinfo": { "name": "db", - "drop": "yes", + "drop": "no", "vgroups": 1 }, "super_tables": [ diff --git a/tests/system-test/insert.json b/tests/system-test/insert.json new file mode 100644 index 0000000000..5dea9eabfe --- /dev/null +++ b/tests/system-test/insert.json @@ -0,0 +1,76 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 16, + "create_table_thread_count": 1, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 0, + "num_of_records_per_req": 10000, + "prepared_rand": 10000, + "chinese": "no", + "databases": [ + { + "dbinfo": { + "name": "db", + "drop": "yes", + "vgroups":4, + "replica": 1, + "precision": "ms" + }, + "super_tables": [ + { + "name": "stb", + "child_table_exists": "no", + "childtable_count": 1000, + "childtable_prefix": "stb_", + "escape_character": "no", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "non_stop_mode": "no", + "line_protocol": "line", + "insert_rows": 100000, + "interlace_rows": 0, + "insert_interval": 0, + "disorder_ratio": 0, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "use_sample_ts": "no", + "tags_file": "", + "columns": [ + { + "type": "FLOAT", + "name": "current", + "count": 4, + "max": 12, + "min": 8 + }, + { "type": "INT", "name": "voltage", "max": 225, "min": 215 }, + { "type": "FLOAT", "name": "phase", "max": 1, "min": 0 } + ], + "tags": [ + { + "type": "TINYINT", + "name": "groupid", + "max": 10, + "min": 1 + }, + { + "name": "location", + "type": "BINARY", + "len": 16, + "values": ["beijing", "shanghai"] + } + ] + } + ] + } + ] +} From 37c122f0a76506d638217a937bd8de31534a1d27 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 9 May 2022 07:03:46 +0000 Subject: [PATCH 10/29] refact --- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index 2dff0f14e1..e9dc98d102 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -326,6 +326,16 @@ static int32_t tsdbMemSkipListCursorMoveTo(SMemSkipListCurosr *pSlc, int64_t ver return 0; } +static int32_t tsdbMemSkipListCursorMoveToFirst(SMemSkipListCurosr *pSlc) { + // TODO + return 0; +} + +static int32_t tsdbMemSkipListCursorMoveToLast(SMemSkipListCurosr *pSlc) { + // TODO + return 0; +} + static int32_t tsdbMemSkipListCursorMoveToNext(SMemSkipListCurosr *pSlc) { // TODO return 0; From f0154c03c91b554bbb3baa0d601128e70c0b7b65 Mon Sep 17 00:00:00 2001 From: dapan Date: Mon, 9 May 2022 16:03:56 +0800 Subject: [PATCH 11/29] stmt auto create table --- source/client/src/clientStmt.c | 9 ++- source/libs/parser/src/parInsert.c | 9 +++ tests/script/api/batchprepare.c | 95 +++++++++++++++++++++++++----- 3 files changed, 95 insertions(+), 18 deletions(-) diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 347debe68d..737282d214 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -29,6 +29,11 @@ int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) { if (STMT_STATUS_EQ(INIT) || STMT_STATUS_EQ(BIND_COL)) { code = TSDB_CODE_TSC_STMT_API_ERROR; } +/* + if ((pStmt->sql.type == STMT_TYPE_MULTI_INSERT) && ()) { + code = TSDB_CODE_TSC_STMT_API_ERROR; + } +*/ break; case STMT_BIND_COL: if (STMT_STATUS_EQ(INIT) || STMT_STATUS_EQ(BIND)) { @@ -513,8 +518,6 @@ int32_t stmtFetchColFields(STscStmt* pStmt, int32_t *fieldNum, TAOS_FIELD** fiel int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int32_t colIdx) { STscStmt* pStmt = (STscStmt*)stmt; - STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_BIND)); - if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { pStmt->bInfo.needParse = false; } @@ -532,6 +535,8 @@ int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int32_t colIdx) { STMT_ERR_RET(stmtParseSql(pStmt)); } + STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_BIND)); + if (STMT_TYPE_QUERY == pStmt->sql.type) { if (NULL == pStmt->sql.pQueryPlan) { STMT_ERR_RET(getQueryPlan(pStmt->exec.pRequest, pStmt->sql.pQuery, &pStmt->sql.nodeList)); diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index ba39ba6986..64d2934282 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -241,6 +241,15 @@ static int32_t getTableMetaImpl(SInsertParseContext* pCxt, SToken* pTname, bool SParseContext* pBasicCtx = pCxt->pComCxt; SName name = {0}; createSName(&name, pTname, pBasicCtx->acctId, pBasicCtx->db, &pCxt->msg); + + char dbFname[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(&name, dbFname); + + bool pass = false; + CHECK_CODE(catalogChkAuth(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, pBasicCtx->pUser, dbFname, AUTH_TYPE_WRITE, &pass)); + if (!pass) { + return TSDB_CODE_PAR_PERMISSION_DENIED; + } if (isStb) { CHECK_CODE(catalogGetSTableMeta(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &pCxt->pTableMeta)); diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index ba9fa4638a..f3fab0c965 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -104,7 +104,7 @@ typedef struct { int32_t colNum; int32_t *colList; // full table column list int32_t testType; - bool prepareStb; + bool autoCreateTbl; bool fullCol; int32_t (*runFn)(TAOS_STMT*, TAOS*); int32_t tblNum; @@ -118,8 +118,8 @@ typedef struct { } CaseCfg; CaseCfg gCase[] = { - {"insert:MBSE1-FULL", tListLen(shortColList), shortColList, TTYPE_INSERT, false, true, insertMBSETest1, 1, 10, 10, 0, 0, 0, 1, -1}, - {"insert:MBSE1-FULL", tListLen(shortColList), shortColList, TTYPE_INSERT, false, true, insertMBSETest1, 10, 100, 10, 0, 0, 0, 1, -1}, + {"insert:MBSE0-FULL", tListLen(shortColList), shortColList, TTYPE_INSERT, false, true, insertMBSETest1, 1, 10, 10, 0, 0, 0, 1, -1}, + {"insert:MBSE0-FULL", tListLen(shortColList), shortColList, TTYPE_INSERT, false, true, insertMBSETest1, 10, 100, 10, 0, 0, 0, 1, -1}, {"insert:MBSE1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBSETest1, 10, 10, 2, 0, 0, 0, 1, -1}, {"insert:MBSE1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBSETest1, 10, 10, 2, 12, 0, 0, 1, -1}, @@ -166,6 +166,7 @@ typedef struct { bool printCreateTblSql; bool printQuerySql; bool printStmtSql; + bool autoCreateTbl; int32_t rowNum; //row num for one table int32_t bindColNum; int32_t bindTagNum; @@ -189,6 +190,35 @@ CaseCtrl gCaseCtrl = { // default .printCreateTblSql = false, .printQuerySql = true, .printStmtSql = true, + .autoCreateTbl = false, + .rowNum = 0, + .bindColNum = 0, + .bindTagNum = 0, + .bindRowNum = 0, + .bindColTypeNum = 0, + .bindColTypeList = NULL, + .bindTagTypeNum = 0, + .bindTagTypeList = NULL, + .optrIdxListNum = 0, + .optrIdxList = NULL, + .checkParamNum = false, + .printRes = true, + .runTimes = 0, + .caseIdx = -1, + .caseNum = -1, + .caseRunIdx = -1, + .caseRunNum = -1, +}; +#endif + + +#if 1 +CaseCtrl gCaseCtrl = { // default + .bindNullNum = 0, + .printCreateTblSql = true, + .printQuerySql = true, + .printStmtSql = true, + .autoCreateTbl = false, .rowNum = 0, .bindColNum = 0, .bindTagNum = 0, @@ -236,12 +266,13 @@ CaseCtrl gCaseCtrl = { // query case with specified col&oper }; #endif -#if 1 +#if 0 CaseCtrl gCaseCtrl = { // query case with specified col&oper .bindNullNum = 0, .printCreateTblSql = true, .printQuerySql = true, .printStmtSql = true, + .autoCreateTbl = true, .rowNum = 0, .bindColNum = 0, .bindTagNum = 0, @@ -1237,6 +1268,15 @@ void bpCheckQueryResult(TAOS_STMT *stmt, TAOS *taos, char *stmtSql, TAOS_MULTI_B printf("***sql res num match stmt res num %d\n", stmtResNum); } +int32_t bpSetTableNameTags(BindData *data, int32_t tblIdx, char *tblName, TAOS_STMT *stmt) { + if (gCurCase->bindTagNum > 0) { + return taos_stmt_set_tbname_tags(stmt, tblName, data->pTags + tblIdx * gCurCase->bindTagNum); + } else { + return taos_stmt_set_tbname(stmt, tblName); + } +} + + /* prepare [settbname [bind add]] exec */ int insertMBSETest1(TAOS_STMT *stmt, TAOS *taos) { BindData data = {0}; @@ -1255,7 +1295,7 @@ int insertMBSETest1(TAOS_STMT *stmt, TAOS *taos) { if (gCurCase->tblNum > 1) { char buf[32]; sprintf(buf, "t%d", t); - code = taos_stmt_set_tbname(stmt, buf); + code = bpSetTableNameTags(&data, t, buf, stmt); if (code != 0){ printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); exit(1); @@ -1312,7 +1352,7 @@ int insertMBSETest2(TAOS_STMT *stmt, TAOS *taos) { if (gCurCase->tblNum > 1) { char buf[32]; sprintf(buf, "t%d", t); - code = taos_stmt_set_tbname(stmt, buf); + code = bpSetTableNameTags(&data, t, buf, stmt); if (code != 0){ printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); exit(1); @@ -1365,7 +1405,7 @@ int insertMBMETest1(TAOS_STMT *stmt, TAOS *taos) { if (gCurCase->tblNum > 1) { char buf[32]; sprintf(buf, "t%d", t); - code = taos_stmt_set_tbname(stmt, buf); + code = bpSetTableNameTags(&data, t, buf, stmt); if (code != 0){ printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); exit(1); @@ -1419,7 +1459,7 @@ int insertMBMETest2(TAOS_STMT *stmt, TAOS *taos) { if (gCurCase->tblNum > 1) { char buf[32]; sprintf(buf, "t%d", t); - code = taos_stmt_set_tbname(stmt, buf); + code = bpSetTableNameTags(&data, t, buf, stmt); if (code != 0){ printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); exit(1); @@ -1473,7 +1513,7 @@ int insertMBMETest3(TAOS_STMT *stmt, TAOS *taos) { if (gCurCase->tblNum > 1) { char buf[32]; sprintf(buf, "t%d", t); - code = taos_stmt_set_tbname(stmt, buf); + code = bpSetTableNameTags(&data, t, buf, stmt); if (code != 0){ printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); exit(1); @@ -1488,7 +1528,7 @@ int insertMBMETest3(TAOS_STMT *stmt, TAOS *taos) { if (gCurCase->tblNum > 1) { char buf[32]; sprintf(buf, "t%d", t); - code = taos_stmt_set_tbname(stmt, buf); + code = bpSetTableNameTags(&data, t, buf, stmt); if (code != 0){ printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); exit(1); @@ -1540,7 +1580,7 @@ int insertMBMETest4(TAOS_STMT *stmt, TAOS *taos) { if (gCurCase->tblNum > 1) { char buf[32]; sprintf(buf, "t%d", t); - code = taos_stmt_set_tbname(stmt, buf); + code = bpSetTableNameTags(&data, t, buf, stmt); if (code != 0){ printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); exit(1); @@ -1596,7 +1636,7 @@ int insertMPMETest1(TAOS_STMT *stmt, TAOS *taos) { if (gCurCase->tblNum > 1) { char buf[32]; sprintf(buf, "t%d", t); - code = taos_stmt_set_tbname(stmt, buf); + code = bpSetTableNameTags(&data, t, buf, stmt); if (code != 0){ printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); exit(1); @@ -1640,11 +1680,12 @@ int insertMPMETest1(TAOS_STMT *stmt, TAOS *taos) { return 0; } + /* [prepare [settbnametag [bind add] exec]] */ int insertAUTOTest1(TAOS_STMT *stmt, TAOS *taos) { int32_t loop = 0; - while (gCurCase->bindColNum >= 2) { + while (gCurCase->bindTagNum > 0 && gCurCase->bindColNum > 0) { BindData data = {0}; prepareInsertData(&data); @@ -3316,10 +3357,23 @@ int32_t runCase(TAOS *taos, int32_t caseIdx, int32_t caseRunIdx, bool silent) { if (gCaseCtrl.rowNum) { gCurCase->rowNum = gCaseCtrl.rowNum; } + + if (gCaseCtrl.autoCreateTbl) { + if (gCurCase->testType == TTYPE_INSERT && gCurCase->tblNum > 1) { + gCurCase->autoCreateTbl = true; + if (gCurCase->bindTagNum <= 0) { + gCurCase->bindTagNum = gCurCase->colNum; + } + } else { + return 1; + } + } if (gCurCase->fullCol) { gCurCase->bindColNum = gCurCase->colNum; - gCurCase->bindTagNum = gCurCase->colNum; + if (gCurCase->autoCreateTbl) { + gCurCase->bindTagNum = gCurCase->colNum; + } } gCurCase->bindNullNum = gCaseCtrl.bindNullNum; @@ -3350,7 +3404,7 @@ int32_t runCase(TAOS *taos, int32_t caseIdx, int32_t caseRunIdx, bool silent) { totalUs = 0; for (int32_t n = 0; n < gCurCase->runTimes; ++n) { if (gCurCase->preCaseIdx < 0) { - prepare(taos, gCurCase->colNum, gCurCase->colList, gCurCase->prepareStb); + prepare(taos, gCurCase->colNum, gCurCase->colList, gCurCase->autoCreateTbl); } beginUs = taosGetTimestampUs(); @@ -3411,10 +3465,18 @@ void* runCaseList(TAOS *taos) { } void runAll(TAOS *taos) { +/* strcpy(gCaseCtrl.caseCatalog, "Normal Test"); printf("%s Begin\n", gCaseCtrl.caseCatalog); runCaseList(taos); +*/ + strcpy(gCaseCtrl.caseCatalog, "Auto Create Table Test"); + gCaseCtrl.autoCreateTbl = true; + printf("%s Begin\n", gCaseCtrl.caseCatalog); + runCaseList(taos); + gCaseCtrl.autoCreateTbl = false; +/* strcpy(gCaseCtrl.caseCatalog, "Null Test"); printf("%s Begin\n", gCaseCtrl.caseCatalog); gCaseCtrl.bindNullNum = 1; @@ -3458,7 +3520,8 @@ void runAll(TAOS *taos) { gCaseCtrl.bindColTypeNum = tListLen(bindColTypeList); gCaseCtrl.bindColTypeList = bindColTypeList; runCaseList(taos); - +*/ + printf("All Test End\n"); } From 0c1ac7a3674e923c0579826a1bc8362e86926ac7 Mon Sep 17 00:00:00 2001 From: dapan Date: Mon, 9 May 2022 17:01:55 +0800 Subject: [PATCH 12/29] stmt test --- source/libs/parser/src/parInsertData.c | 2 +- tests/script/api/batchprepare.c | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index c5c4463a95..5fa2961474 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -469,7 +469,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p // the maximum expanded size in byte when a row-wise data is converted to SDataRow format int32_t expandSize = isRawPayload ? getRowExpandSize(pOneTableBlock->pTableMeta) : 0; int64_t destSize = dataBuf->size + pOneTableBlock->size + pBlocks->numOfRows * expandSize + - sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta); + sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta) + pOneTableBlock->createTbReqLen; if (dataBuf->nAllocSize < destSize) { dataBuf->nAllocSize = (uint32_t)(destSize * 1.5); diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index f3fab0c965..8169e6c503 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -233,9 +233,9 @@ CaseCtrl gCaseCtrl = { // default .printRes = true, .runTimes = 0, .caseIdx = -1, - .caseNum = -1, - .caseRunIdx = -1, - .caseRunNum = -1, + .caseNum = 1, + .caseRunIdx = 11, + .caseRunNum = 1, }; #endif From db998d29b6091009aa5657ffe711c500ab631c44 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 9 May 2022 09:43:15 +0000 Subject: [PATCH 13/29] refact: tdata --- include/common/tdata.h | 45 ++ include/common/tdataformat.h | 410 +----------------- include/common/trow.h | 76 +--- source/common/src/tdata.c | 14 + source/common/src/tdataformat.c | 450 -------------------- source/dnode/vnode/src/tsdb/tsdbCommit.c | 58 +-- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 32 +- 7 files changed, 96 insertions(+), 989 deletions(-) create mode 100644 include/common/tdata.h create mode 100644 source/common/src/tdata.c diff --git a/include/common/tdata.h b/include/common/tdata.h new file mode 100644 index 0000000000..9c4d33ed02 --- /dev/null +++ b/include/common/tdata.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TDATA_H_ +#define _TD_TDATA_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "os.h" + +typedef struct STaosData TDATA, tdata_t; + +typedef enum { + TAOS_META_STABLE_DATA = 0, // super table meta + TAOS_META_TABLE_DATA, // non-super table meta + TAOS_TS_ROW_DATA, // row time-series data + TAOS_TS_COL_DATA, // col time-series data + TAOS_DATA_MAX +} ETaosDataT; + +struct STaosData { + ETaosDataT type; + uint32_t nPayload; + uint8_t *pPayload; +}; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_TDATA_H_*/ \ No newline at end of file diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index e94be797b9..1f3b787538 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -150,29 +150,6 @@ int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags, STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); // ----------------- Semantic timestamp key definition -#ifdef TD_2_0 - -typedef uint64_t TKEY; - -#define TKEY_INVALID UINT64_MAX -#define TKEY_NULL TKEY_INVALID -#define TKEY_NEGATIVE_FLAG (((TKEY)1) << 63) -#define TKEY_DELETE_FLAG (((TKEY)1) << 62) -#define TKEY_VALUE_FILTER (~(TKEY_NEGATIVE_FLAG | TKEY_DELETE_FLAG)) - -#define TKEY_IS_NEGATIVE(tkey) (((tkey)&TKEY_NEGATIVE_FLAG) != 0) -#define TKEY_IS_DELETED(tkey) (((tkey)&TKEY_DELETE_FLAG) != 0) -#define tdSetTKEYDeleted(tkey) ((tkey) | TKEY_DELETE_FLAG) -#define tdGetTKEY(key) (((TKEY)TABS(key)) | (TKEY_NEGATIVE_FLAG & (TKEY)(key))) -#define tdGetKey(tkey) (((TSKEY)((tkey)&TKEY_VALUE_FILTER)) * (TKEY_IS_NEGATIVE(tkey) ? -1 : 1)) - -#define MIN_TS_KEY ((TSKEY)0x8000000000000001) -#define MAX_TS_KEY ((TSKEY)0x3fffffffffffffff) - -#define TD_TO_TKEY(key) tdGetTKEY(((key) < MIN_TS_KEY) ? MIN_TS_KEY : (((key) > MAX_TS_KEY) ? MAX_TS_KEY : key)) - -#else - // typedef uint64_t TKEY; #define TKEY TSKEY @@ -192,8 +169,6 @@ typedef uint64_t TKEY; #define TD_TO_TKEY(key) tdGetTKEY(((key) < MIN_TS_KEY) ? MIN_TS_KEY : (((key) > MAX_TS_KEY) ? MAX_TS_KEY : key)) -#endif - static FORCE_INLINE TKEY keyToTkey(TSKEY key) { TSKEY lkey = key; if (key > MAX_TS_KEY) { @@ -218,157 +193,6 @@ static FORCE_INLINE int32_t tkeyComparFn(const void *tkey1, const void *tkey2) { } } -#if 0 -// ----------------- Data row structure - -/* A data row, the format is like below: - * |<------------------------------------------------ len ---------------------------------->| - * |<-- Head -->|<--------- flen -------------->| | - * +---------------------+---------------------------------+---------------------------------+ - * | uint16_t | int16_t | | | - * +----------+----------+---------------------------------+---------------------------------+ - * | len | sversion | First part | Second part | - * +----------+----------+---------------------------------+---------------------------------+ - * - * NOTE: timestamp in this row structure is TKEY instead of TSKEY - */ -typedef void *SDataRow; - -#define TD_DATA_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t)) - -#define dataRowLen(r) (*(TDRowLenT *)(r)) // 0~65535 -#define dataRowEnd(r) POINTER_SHIFT(r, dataRowLen(r)) -#define dataRowVersion(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(int16_t))) -#define dataRowTuple(r) POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE) -#define dataRowTKey(r) (*(TKEY *)(dataRowTuple(r))) -#define dataRowKey(r) tdGetKey(dataRowTKey(r)) -#define dataRowSetLen(r, l) (dataRowLen(r) = (l)) -#define dataRowSetVersion(r, v) (dataRowVersion(r) = (v)) -#define dataRowCpy(dst, r) memcpy((dst), (r), dataRowLen(r)) -#define dataRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_DATA_ROW_HEAD_SIZE) -#define dataRowDeleted(r) TKEY_IS_DELETED(dataRowTKey(r)) - -SDataRow tdNewDataRowFromSchema(STSchema *pSchema); -void tdFreeDataRow(SDataRow row); -void tdInitDataRow(SDataRow row, STSchema *pSchema); -SDataRow tdDataRowDup(SDataRow row); - -// offset here not include dataRow header length -static FORCE_INLINE int32_t tdAppendDataColVal(SDataRow row, const void *value, bool isCopyVarData, int8_t type, - int32_t offset) { - assert(value != NULL); - int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE; - - if (IS_VAR_DATA_TYPE(type)) { - *(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row); - if (isCopyVarData) { - memcpy(POINTER_SHIFT(row, dataRowLen(row)), value, varDataTLen(value)); - } - dataRowLen(row) += varDataTLen(value); - } else { - if (offset == 0) { - assert(type == TSDB_DATA_TYPE_TIMESTAMP); - TKEY tvalue = tdGetTKEY(*(TSKEY *)value); - memcpy(POINTER_SHIFT(row, toffset), (const void *)(&tvalue), TYPE_BYTES[type]); - } else { - memcpy(POINTER_SHIFT(row, toffset), value, TYPE_BYTES[type]); - } - } - - return 0; -} - -// offset here not include dataRow header length -static FORCE_INLINE int32_t tdAppendColVal(SDataRow row, const void *value, int8_t type, int32_t offset) { - return tdAppendDataColVal(row, value, true, type, offset); -} - -// NOTE: offset here including the header size -static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t offset) { - if (IS_VAR_DATA_TYPE(type)) { - return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset)); - } else { - return POINTER_SHIFT(row, offset); - } -} - -static FORCE_INLINE void *tdGetPtrToCol(SDataRow row, STSchema *pSchema, int32_t idx) { - return POINTER_SHIFT(row, TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset); -} - -static FORCE_INLINE void *tdGetColOfRowBySchema(SDataRow row, STSchema *pSchema, int32_t idx) { - int16_t offset = TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset; - int8_t type = pSchema->columns[idx].type; - - return tdGetRowDataOfCol(row, type, offset); -} - -static FORCE_INLINE bool tdIsColOfRowNullBySchema(SDataRow row, STSchema *pSchema, int32_t idx) { - int16_t offset = TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset; - int8_t type = pSchema->columns[idx].type; - - return isNull(tdGetRowDataOfCol(row, type, offset), type); -} - -static FORCE_INLINE void tdSetColOfRowNullBySchema(SDataRow row, STSchema *pSchema, int32_t idx) { - int16_t offset = TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset; - int8_t type = pSchema->columns[idx].type; - int16_t bytes = pSchema->columns[idx].bytes; - - setNull(tdGetRowDataOfCol(row, type, offset), type, bytes); -} - -static FORCE_INLINE void tdCopyColOfRowBySchema(SDataRow dst, STSchema *pDstSchema, int32_t dstIdx, SDataRow src, - STSchema *pSrcSchema, int32_t srcIdx) { - int8_t type = pDstSchema->columns[dstIdx].type; - assert(type == pSrcSchema->columns[srcIdx].type); - void *pData = tdGetPtrToCol(dst, pDstSchema, dstIdx); - void *value = tdGetPtrToCol(src, pSrcSchema, srcIdx); - - switch (type) { - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - *(VarDataOffsetT *)pData = *(VarDataOffsetT *)value; - pData = POINTER_SHIFT(dst, *(VarDataOffsetT *)pData); - value = POINTER_SHIFT(src, *(VarDataOffsetT *)value); - memcpy(pData, value, varDataTLen(value)); - break; - case TSDB_DATA_TYPE_NULL: - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT: - *(uint8_t *)pData = *(uint8_t *)value; - break; - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: - *(uint16_t *)pData = *(uint16_t *)value; - break; - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - *(uint32_t *)pData = *(uint32_t *)value; - break; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - *(uint64_t *)pData = *(uint64_t *)value; - break; - case TSDB_DATA_TYPE_FLOAT: - SET_FLOAT_PTR(pData, value); - break; - case TSDB_DATA_TYPE_DOUBLE: - SET_DOUBLE_PTR(pData, value); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - if (pSrcSchema->columns[srcIdx].colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - *(TSKEY *)pData = tdGetKey(*(TKEY *)value); - } else { - *(TSKEY *)pData = *(TSKEY *)value; - } - break; - default: - memcpy(pData, value, pSrcSchema->columns[srcIdx].bytes); - } -} -#endif // ----------------- Data column structure // SDataCol arrangement: data => bitmap => dataOffset typedef struct SDataCol { @@ -398,29 +222,6 @@ void *dataColSetOffset(SDataCol *pCol, int32_t nEle); bool isNEleNull(SDataCol *pCol, int32_t nEle); -#if 0 -// Get the data pointer from a column-wised data -static FORCE_INLINE const void *tdGetColDataOfRow(SDataCol *pCol, int32_t row) { - if (isAllRowsNull(pCol)) { - return getNullValue(pCol->type); - } - if (IS_VAR_DATA_TYPE(pCol->type)) { - return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]); - } else { - return POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row); - } -} - -static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int32_t rows) { - assert(rows > 0); - - if (IS_VAR_DATA_TYPE(pDataCol->type)) { - return pDataCol->dataOff[rows - 1] + varDataTLen(tdGetColDataOfRow(pDataCol, rows - 1)); - } else { - return TYPE_BYTES[pDataCol->type] * rows; - } -} -#endif typedef struct { col_id_t maxCols; // max number of columns col_id_t numOfCols; // Total number of cols @@ -479,7 +280,8 @@ void tdResetDataCols(SDataCols *pCols); int32_t tdInitDataCols(SDataCols *pCols, STSchema *pSchema); SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData); SDataCols *tdFreeDataCols(SDataCols *pCols); -int32_t tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToMerge, int32_t *pOffset, bool update, TDRowVerT maxVer); +int32_t tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToMerge, int32_t *pOffset, bool update, + TDRowVerT maxVer); // ----------------- K-V data row structure /* |<-------------------------------------- len -------------------------------------------->| @@ -542,54 +344,6 @@ static FORCE_INLINE void *tdGetKVRowIdxOfCol(SKVRow row, int16_t colId) { return taosbsearch(&colId, kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), comparTagId, TD_EQ); } -#if 0 -// offset here not include kvRow header length -static FORCE_INLINE int32_t tdAppendKvColVal(SKVRow row, const void *value, bool isCopyValData, int16_t colId, int8_t type, - int32_t offset) { - assert(value != NULL); - int32_t toffset = offset + TD_KV_ROW_HEAD_SIZE; - SColIdx *pColIdx = (SColIdx *)POINTER_SHIFT(row, toffset); - char * ptr = (char *)POINTER_SHIFT(row, kvRowLen(row)); - - pColIdx->colId = colId; - pColIdx->offset = kvRowLen(row); // offset of pColIdx including the TD_KV_ROW_HEAD_SIZE - - if (IS_VAR_DATA_TYPE(type)) { - if (isCopyValData) { - memcpy(ptr, value, varDataTLen(value)); - } - kvRowLen(row) += varDataTLen(value); - } else { - if (offset == 0) { - assert(type == TSDB_DATA_TYPE_TIMESTAMP); - TKEY tvalue = tdGetTKEY(*(TSKEY *)value); - memcpy(ptr, (void *)(&tvalue), TYPE_BYTES[type]); - } else { - memcpy(ptr, value, TYPE_BYTES[type]); - } - kvRowLen(row) += TYPE_BYTES[type]; - } - - return 0; -} -// NOTE: offset here including the header size -static FORCE_INLINE void *tdGetKvRowDataOfCol(void *row, int32_t offset) { return POINTER_SHIFT(row, offset); } - -static FORCE_INLINE void *tdGetKVRowValOfColEx(SKVRow row, int16_t colId, int32_t *nIdx) { - while (*nIdx < kvRowNCols(row)) { - SColIdx *pColIdx = kvRowColIdxAt(row, *nIdx); - if (pColIdx->colId == colId) { - ++(*nIdx); - return tdGetKvRowDataOfCol(row, pColIdx->offset); - } else if (pColIdx->colId > colId) { - return NULL; - } else { - ++(*nIdx); - } - } - return NULL; -} -#endif // ----------------- K-V data row builder typedef struct { int16_t tCols; @@ -632,166 +386,6 @@ static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t co return 0; } -#if 0 -// ----------------- SMemRow appended with tuple row structure -/* - * |---------|------------------------------------------------- len ---------------------------------->| - * |<-------- Head ------>|<--------- flen -------------->| | - * |---------+---------------------+---------------------------------+---------------------------------+ - * | uint8_t | uint16_t | int16_t | | | - * |---------+----------+----------+---------------------------------+---------------------------------+ - * | flag | len | sversion | First part | Second part | - * +---------+----------+----------+---------------------------------+---------------------------------+ - * - * NOTE: timestamp in this row structure is TKEY instead of TSKEY - */ - -// ----------------- SMemRow appended with extended K-V data row structure -/* |--------------------|------------------------------------------------ len ---------------------------------->| - * |<------------- Head ------------>|<--------- flen -------------->| | - * |--------------------+----------+--------------------------------------------+---------------------------------+ - * | uint8_t | int16_t | uint16_t | int16_t | | | - * |---------+----------+----------+----------+---------------------------------+---------------------------------+ - * | flag | sversion | len | ncols | cols index | data part | - * |---------+----------+----------+----------+---------------------------------+---------------------------------+ - */ - -typedef void *SMemRow; - -#define TD_MEM_ROW_TYPE_SIZE sizeof(uint8_t) -#define TD_MEM_ROW_KV_VER_SIZE sizeof(int16_t) -#define TD_MEM_ROW_KV_TYPE_VER_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_MEM_ROW_KV_VER_SIZE) -#define TD_MEM_ROW_DATA_HEAD_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_DATA_ROW_HEAD_SIZE) -#define TD_MEM_ROW_KV_HEAD_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_MEM_ROW_KV_VER_SIZE + TD_KV_ROW_HEAD_SIZE) - -#define SMEM_ROW_DATA 0x0U // SDataRow -#define SMEM_ROW_KV 0x01U // SKVRow - -#define KVRatioConvert (0.9f) - -#define memRowType(r) ((*(uint8_t *)(r)) & 0x01) - -#define memRowSetType(r, t) ((*(uint8_t *)(r)) = (t)) // set the total byte in case of dirty memory -#define isDataRowT(t) (SMEM_ROW_DATA == (((uint8_t)(t)) & 0x01)) -#define isDataRow(r) (SMEM_ROW_DATA == memRowType(r)) -#define isKvRowT(t) (SMEM_ROW_KV == (((uint8_t)(t)) & 0x01)) -#define isKvRow(r) (SMEM_ROW_KV == memRowType(r)) -#define isUtilizeKVRow(k, d) ((k) < ((d)*KVRatioConvert)) - -#define memRowDataBody(r) POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE) // section after flag -#define memRowKvBody(r) \ - POINTER_SHIFT(r, TD_MEM_ROW_KV_TYPE_VER_SIZE) // section after flag + sversion as to reuse SKVRow - -#define memRowDataLen(r) (*(TDRowLenT *)memRowDataBody(r)) // 0~65535 -#define memRowKvLen(r) (*(TDRowLenT *)memRowKvBody(r)) // 0~65535 - -#define memRowDataTLen(r) \ - ((TDRowLenT)(memRowDataLen(r) + TD_MEM_ROW_TYPE_SIZE)) // using uint32_t/int32_t to store the TLen - -#define memRowKvTLen(r) ((TDRowLenT)(memRowKvLen(r) + TD_MEM_ROW_KV_TYPE_VER_SIZE)) - -#define memRowLen(r) (isDataRow(r) ? memRowDataLen(r) : memRowKvLen(r)) -#define memRowTLen(r) (isDataRow(r) ? memRowDataTLen(r) : memRowKvTLen(r)) // using uint32_t/int32_t to store the TLen - -static FORCE_INLINE char *memRowEnd(SMemRow row) { - if (isDataRow(row)) { - return (char *)dataRowEnd(memRowDataBody(row)); - } else { - return (char *)kvRowEnd(memRowKvBody(row)); - } -} - -#define memRowDataVersion(r) dataRowVersion(memRowDataBody(r)) -#define memRowKvVersion(r) (*(int16_t *)POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE)) -#define memRowVersion(r) (isDataRow(r) ? memRowDataVersion(r) : memRowKvVersion(r)) // schema version -#define memRowSetKvVersion(r, v) (memRowKvVersion(r) = (v)) -#define memRowTuple(r) (isDataRow(r) ? dataRowTuple(memRowDataBody(r)) : kvRowValues(memRowKvBody(r))) - -#define memRowTKey(r) (isDataRow(r) ? dataRowTKey(memRowDataBody(r)) : kvRowTKey(memRowKvBody(r))) -#define memRowKey(r) (isDataRow(r) ? dataRowKey(memRowDataBody(r)) : kvRowKey(memRowKvBody(r))) -#define memRowKeys(r) (isDataRow(r) ? dataRowTuple(memRowDataBody(r)) : kvRowKeys(memRowKvBody(r))) -#define memRowSetTKey(r, k) \ - do { \ - if (isDataRow(r)) { \ - dataRowTKey(memRowDataBody(r)) = (k); \ - } else { \ - kvRowTKey(memRowKvBody(r)) = (k); \ - } \ - } while (0) - -#define memRowSetLen(r, l) (isDataRow(r) ? memRowDataLen(r) = (l) : memRowKvLen(r) = (l)) -#define memRowSetVersion(r, v) (isDataRow(r) ? dataRowSetVersion(memRowDataBody(r), v) : memRowSetKvVersion(r, v)) -#define memRowCpy(dst, r) memcpy((dst), (r), memRowTLen(r)) -#define memRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_MEM_ROW_DATA_HEAD_SIZE) -#define memRowDeleted(r) TKEY_IS_DELETED(memRowTKey(r)) - -SMemRow tdMemRowDup(SMemRow row); -void tdAppendMemRowToDataCol(SMemRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull); - -// NOTE: offset here including the header size -static FORCE_INLINE void *tdGetMemRowDataOfCol(void *row, int16_t colId, int8_t colType, uint16_t offset) { - if (isDataRow(row)) { - return tdGetRowDataOfCol(memRowDataBody(row), colType, offset); - } else { - return tdGetKVRowValOfCol(memRowKvBody(row), colId); - } -} - -/** - * NOTE: - * 1. Applicable to scan columns one by one - * 2. offset here including the header size - */ -static FORCE_INLINE void *tdGetMemRowDataOfColEx(void *row, int16_t colId, int8_t colType, int32_t offset, - int32_t *kvNIdx) { - if (isDataRow(row)) { - return tdGetRowDataOfCol(memRowDataBody(row), colType, offset); - } else { - return tdGetKVRowValOfColEx(memRowKvBody(row), colId, kvNIdx); - } -} - -static FORCE_INLINE int32_t tdAppendMemRowColVal(SMemRow row, const void *value, bool isCopyVarData, int16_t colId, - int8_t type, int32_t offset) { - if (isDataRow(row)) { - tdAppendDataColVal(memRowDataBody(row), value, isCopyVarData, type, offset); - } else { - tdAppendKvColVal(memRowKvBody(row), value, isCopyVarData, colId, type, offset); - } - return 0; -} - -// make sure schema->flen appended for SDataRow -static FORCE_INLINE int32_t tdGetColAppendLen(uint8_t rowType, const void *value, int8_t colType) { - int32_t len = 0; - if (IS_VAR_DATA_TYPE(colType)) { - len += varDataTLen(value); - if (rowType == SMEM_ROW_KV) { - len += sizeof(SColIdx); - } - } else { - if (rowType == SMEM_ROW_KV) { - len += TYPE_BYTES[colType]; - len += sizeof(SColIdx); - } - } - return len; -} - -typedef struct { - int16_t colId; - uint8_t colType; - char * colVal; -} SColInfo; - -static FORCE_INLINE void setSColInfo(SColInfo *colInfo, int16_t colId, uint8_t colType, char *colVal) { - colInfo->colId = colId; - colInfo->colType = colType; - colInfo->colVal = colVal; -} - -SMemRow mergeTwoMemRows(void *buffer, SMemRow row1, SMemRow row2, STSchema *pSchema1, STSchema *pSchema2); -#endif #ifdef __cplusplus } diff --git a/include/common/trow.h b/include/common/trow.h index 0d34c6e49f..ebce0b085d 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -165,15 +165,15 @@ typedef struct { #define TD_ROW_HEAD_LEN (sizeof(STSRow)) #define TD_ROW_NCOLS_LEN (sizeof(col_id_t)) -#define TD_ROW_INFO(r) ((r)->info) -#define TD_ROW_TYPE(r) ((r)->type) -#define TD_ROW_DELETE(r) ((r)->del) -#define TD_ROW_ENDIAN(r) ((r)->endian) -#define TD_ROW_SVER(r) ((r)->sver) -#define TD_ROW_NCOLS(r) ((r)->data) // only valid for SKvRow -#define TD_ROW_DATA(r) ((r)->data) -#define TD_ROW_LEN(r) ((r)->len) -#define TD_ROW_KEY(r) ((r)->ts) +#define TD_ROW_INFO(r) ((r)->info) +#define TD_ROW_TYPE(r) ((r)->type) +#define TD_ROW_DELETE(r) ((r)->del) +#define TD_ROW_ENDIAN(r) ((r)->endian) +#define TD_ROW_SVER(r) ((r)->sver) +#define TD_ROW_NCOLS(r) ((r)->data) // only valid for SKvRow +#define TD_ROW_DATA(r) ((r)->data) +#define TD_ROW_LEN(r) ((r)->len) +#define TD_ROW_KEY(r) ((r)->ts) // #define TD_ROW_VER(r) ((r)->ver) #define TD_ROW_KEY_ADDR(r) (r) @@ -1410,64 +1410,6 @@ static void tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag) { } printf("\n"); } -#ifdef TROW_ORIGIN_HZ -typedef struct { - uint32_t nRows; - char rows[]; -} STSRowBatch; - -static void tdSRowPrint(STSRow *row) { - printf("type:%d, del:%d, sver:%d\n", row->type, row->del, row->sver); - printf("isDeleted:%s, isTpRow:%s, isKvRow:%s\n", TD_BOOL_STR(TD_ROW_IS_DELETED(row)), TD_BOOL_STR(TD_IS_TP_ROW(row)), - TD_BOOL_STR(TD_IS_KV_ROW(row))); -} - -typedef enum { - /// tuple row builder - TD_TP_ROW_BUILDER = 0, - /// kv row builder - TD_KV_ROW_BUILDER, - /// self-determined row builder - TD_SD_ROW_BUILDER -} ERowBbuilderT; - -typedef struct { - /// row builder type - ERowBbuilderT type; - /// buffer writer - SBufferWriter bw; - /// target row - STSRow *pRow; -} STSRowBuilder; - -typedef struct { - STSchema *pSchema; - STSRow *pRow; -} STSRowReader; - -typedef struct { - uint32_t it; - STSRowBatch *pRowBatch; -} STSRowBatchIter; - -// STSRowBuilder -#define trbInit(rt, allocator, endian, target, size) \ - { .type = (rt), .bw = tbufInitWriter(allocator, endian), .pRow = (target) } -void trbSetRowInfo(STSRowBuilder *pRB, bool del, uint16_t sver); -void trbSetRowVersion(STSRowBuilder *pRB, uint64_t ver); -void trbSetRowTS(STSRowBuilder *pRB, TSKEY ts); -int32_t trbWriteCol(STSRowBuilder *pRB, void *pData, col_id_t cid); - -// STSRowReader -#define tRowReaderInit(schema, row) \ - { .schema = (schema), .row = (row) } -int32_t tRowReaderRead(STSRowReader *pRowReader, col_id_t cid, void *target, uint64_t size); - -// STSRowBatchIter -#define tRowBatchIterInit(pRB) \ - { .it = 0, .pRowBatch = (pRB) } -const STSRow *tRowBatchIterNext(STSRowBatchIter *pRowBatchIter); -#endif #ifdef __cplusplus } diff --git a/source/common/src/tdata.c b/source/common/src/tdata.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/source/common/src/tdata.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index e659b9ec04..5d893fe398 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -129,50 +129,6 @@ void *tdDecodeSchema(void *buf, STSchema **pRSchema) { return buf; } -#if 0 -int32_t tEncodeSTColumn(SCoder *pEncoder, const STColumn *pCol) { - if (tEncodeI16(pEncoder, pCol->colId) < 0) return -1; - if (tEncodeI8(pEncoder, pCol->type) < 0) return -1; - if (tEncodeI8(pEncoder, pCol->sma) < 0) return -1; - if (tEncodeI32(pEncoder, pCol->bytes) < 0) return -1; - if (tEncodeI32(pEncoder, pCol->offset) < 0) return -1; - return pEncoder->pos; -} - -int32_t tDecodeSTColumn(SCoder *pDecoder, STColumn *pCol) { - if (tDecodeI16(pDecoder, &pCol->colId) < 0) return -1; - if (tDecodeI8(pDecoder, &pCol->type) < 0) return -1; - if (tDecodeI8(pDecoder, &pCol->sma) < 0) return -1; - if (tDecodeI32(pDecoder, &pCol->bytes) < 0) return -1; - if (tDecodeI32(pDecoder, &pCol->offset) < 0) return -1; - return 0; -} - -int32_t tEncodeSchema(SCoder *pEncoder, const STSchema *pSchema) { - if (tEncodeI32(pEncoder, pSchema->numOfCols) < 0) return -1; - if (tEncodeI16(pEncoder, pSchema->version) < 0) return -1; - if (tEncodeU16(pEncoder, pSchema->flen) < 0) return -1; - if (tEncodeI32(pEncoder, pSchema->vlen) < 0) return -1; - if (tEncodeI32(pEncoder, pSchema->tlen) < 0) return -1; - - for (int32_t i = 0; i < schemaNCols(pSchema); i++) { - const STColumn *pCol = schemaColAt(pSchema, i); - if (tEncodeSTColumn(pEncoder, pCol) < 0) return -1; - } - return 0; -} - -int32_t tDecodeSchema(SCoder *pDecoder, STSchema *pSchema) { - if (tDecodeI32(pDecoder, &pSchema->numOfCols) < 0) return -1; - if (tDecodeI16(pDecoder, &pSchema->version) < 0) return -1; - if (tDecodeU16(pDecoder, &pSchema->flen) < 0) return -1; - if (tDecodeI32(pDecoder, &pSchema->vlen) < 0) return -1; - if (tDecodeI32(pDecoder, &pSchema->tlen) < 0) return -1; - - return 0; -} -#endif - int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) { if (pBuilder == NULL) return -1; @@ -260,49 +216,6 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) { return pSchema; } -#if 0 -/** - * Initialize a data row - */ -void tdInitDataRow(SDataRow row, STSchema *pSchema) { - dataRowSetLen(row, TD_DATA_ROW_HEAD_SIZE + schemaFLen(pSchema)); - dataRowSetVersion(row, schemaVersion(pSchema)); -} - -SDataRow tdNewDataRowFromSchema(STSchema *pSchema) { - int32_t size = dataRowMaxBytesFromSchema(pSchema); - - SDataRow row = taosMemoryMalloc(size); - if (row == NULL) return NULL; - - tdInitDataRow(row, pSchema); - return row; -} - -/** - * Free the SDataRow object - */ -void tdFreeDataRow(SDataRow row) { - if (row) taosMemoryFree(row); -} - -SDataRow tdDataRowDup(SDataRow row) { - SDataRow trow = taosMemoryMalloc(dataRowLen(row)); - if (trow == NULL) return NULL; - - dataRowCpy(trow, row); - return trow; -} - -SMemRow tdMemRowDup(SMemRow row) { - SMemRow trow = taosMemoryMalloc(memRowTLen(row)); - if (trow == NULL) return NULL; - - memRowCpy(trow, row); - return trow; -} -#endif - void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints) { pDataCol->type = colType(pCol); pDataCol->colId = colColId(pCol); @@ -312,39 +225,6 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints) { pDataCol->len = 0; } -#if 0 -// value from timestamp should be TKEY here instead of TSKEY -int dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints) { - ASSERT(pCol != NULL && value != NULL); - - if (isAllRowsNull(pCol)) { - if (isNull(value, pCol->type)) { - // all null value yet, just return - return 0; - } - - if (tdAllocMemForCol(pCol, maxPoints) < 0) return -1; - if (numOfRows > 0) { - // Find the first not null value, fill all previouse values as NULL - dataColSetNEleNull(pCol, numOfRows); - } - } - - if (IS_VAR_DATA_TYPE(pCol->type)) { - // set offset - pCol->dataOff[numOfRows] = pCol->len; - // Copy data - memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value)); - // Update the length - pCol->len += varDataTLen(value); - } else { - ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows); - memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes); - pCol->len += pCol->bytes; - } - return 0; -} -#endif static FORCE_INLINE const void *tdGetColDataOfRowUnsafe(SDataCol *pCol, int row) { if (IS_VAR_DATA_TYPE(pCol->type)) { return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]); @@ -361,31 +241,6 @@ bool isNEleNull(SDataCol *pCol, int nEle) { return true; } -#if 0 -static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index) { - if (IS_VAR_DATA_TYPE(pCol->type)) { - pCol->dataOff[index] = pCol->len; - char *ptr = POINTER_SHIFT(pCol->pData, pCol->len); - setVardataNull(ptr, pCol->type); - pCol->len += varDataTLen(ptr); - } else { - setNull(POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * index), pCol->type, pCol->bytes); - pCol->len += TYPE_BYTES[pCol->type]; - } -} - -static void dataColSetNEleNull(SDataCol *pCol, int nEle, int8_t bitmapMode) { - if (IS_VAR_DATA_TYPE(pCol->type)) { - pCol->len = 0; - for (int i = 0; i < nEle; ++i) { - dataColSetNullAt(pCol, i); - } - } else { - setNullN(pCol->pData, pCol->type, pCol->bytes, nEle); - pCol->len = TYPE_BYTES[pCol->type] * nEle; - } -} -#endif void *dataColSetOffset(SDataCol *pCol, int nEle) { ASSERT(((pCol->type == TSDB_DATA_TYPE_BINARY) || (pCol->type == TSDB_DATA_TYPE_NCHAR))); @@ -483,42 +338,6 @@ SDataCols *tdFreeDataCols(SDataCols *pCols) { return NULL; } -#if 0 -SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { - SDataCols *pRet = tdNewDataCols(pDataCols->maxCols, pDataCols->maxPoints); - if (pRet == NULL) return NULL; - - pRet->numOfCols = pDataCols->numOfCols; - pRet->sversion = pDataCols->sversion; - if (keepData) pRet->numOfRows = pDataCols->numOfRows; - - for (int i = 0; i < pDataCols->numOfCols; i++) { - pRet->cols[i].type = pDataCols->cols[i].type; - pRet->cols[i].bitmap = pDataCols->cols[i].bitmap; - pRet->cols[i].colId = pDataCols->cols[i].colId; - pRet->cols[i].bytes = pDataCols->cols[i].bytes; - pRet->cols[i].offset = pDataCols->cols[i].offset; - - if (keepData) { - if (pDataCols->cols[i].len > 0) { - if (tdAllocMemForCol(&pRet->cols[i], pRet->maxPoints) < 0) { - tdFreeDataCols(pRet); - return NULL; - } - pRet->cols[i].len = pDataCols->cols[i].len; - memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pDataCols->cols[i].len); - if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) { - int dataOffSize = sizeof(VarDataOffsetT) * pDataCols->maxPoints; - memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, dataOffSize); - } - } - } - } - - return pRet; -} -#endif - void tdResetDataCols(SDataCols *pCols) { if (pCols != NULL) { pCols->numOfRows = 0; @@ -528,180 +347,6 @@ void tdResetDataCols(SDataCols *pCols) { } } } -#if 0 -static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull) { - ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < dataRowKey(row)); - - int rcol = 0; - int dcol = 0; - - while (dcol < pCols->numOfCols) { - bool setCol = 0; - SDataCol *pDataCol = &(pCols->cols[dcol]); - if (rcol >= schemaNCols(pSchema)) { - dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints); - dcol++; - continue; - } - - STColumn *pRowCol = schemaColAt(pSchema, rcol); - if (pRowCol->colId == pDataCol->colId) { - void *value = tdGetRowDataOfCol(row, pRowCol->type, pRowCol->offset + TD_DATA_ROW_HEAD_SIZE); - if(!isNull(value, pDataCol->type)) setCol = 1; - dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints); - dcol++; - rcol++; - } else if (pRowCol->colId < pDataCol->colId) { - rcol++; - } else { - if(forceSetNull || setCol) { - dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints); - } - dcol++; - } - } - pCols->numOfRows++; -} - -static void tdAppendKVRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull) { - ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < kvRowKey(row)); - - int rcol = 0; - int dcol = 0; - - int nRowCols = kvRowNCols(row); - - while (dcol < pCols->numOfCols) { - bool setCol = 0; - SDataCol *pDataCol = &(pCols->cols[dcol]); - if (rcol >= nRowCols || rcol >= schemaNCols(pSchema)) { - dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints); - ++dcol; - continue; - } - - SColIdx *colIdx = kvRowColIdxAt(row, rcol); - - if (colIdx->colId == pDataCol->colId) { - void *value = tdGetKvRowDataOfCol(row, colIdx->offset); - if(!isNull(value, pDataCol->type)) setCol = 1; - dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints); - ++dcol; - ++rcol; - } else if (colIdx->colId < pDataCol->colId) { - ++rcol; - } else { - if(forceSetNull || setCol) { - dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints); - } - ++dcol; - } - } - pCols->numOfRows++; -} - -void tdAppendMemRowToDataCol(SMemRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull) { - if (isDataRow(row)) { - tdAppendDataRowToDataCol(memRowDataBody(row), pSchema, pCols, forceSetNull); - } else if (isKvRow(row)) { - tdAppendKVRowToDataCol(memRowKvBody(row), pSchema, pCols, forceSetNull); - } else { - ASSERT(0); - } -} - -int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset, bool forceSetNull) { - ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfRows); - ASSERT(target->numOfCols == source->numOfCols); - int offset = 0; - - if (pOffset == NULL) { - pOffset = &offset; - } - - SDataCols *pTarget = NULL; - - if ((target->numOfRows == 0) || (dataColsKeyLast(target) < dataColsKeyAtRow(source, *pOffset))) { // No overlap - ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints); - for (int i = 0; i < rowsToMerge; i++) { - for (int j = 0; j < source->numOfCols; j++) { - if (source->cols[j].len > 0 || target->cols[j].len > 0) { - dataColAppendVal(target->cols + j, tdGetColDataOfRow(source->cols + j, i + (*pOffset)), target->numOfRows, - target->maxPoints); - } - } - target->numOfRows++; - } - (*pOffset) += rowsToMerge; - } else { - pTarget = tdDupDataCols(target, true); - if (pTarget == NULL) goto _err; - - int iter1 = 0; - tdMergeTwoDataCols(target, pTarget, &iter1, pTarget->numOfRows, source, pOffset, source->numOfRows, - pTarget->numOfRows + rowsToMerge, forceSetNull); - } - - tdFreeDataCols(pTarget); - return 0; - -_err: - tdFreeDataCols(pTarget); - return -1; -} - -// src2 data has more priority than src1 -static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, - int limit2, int tRows, bool forceSetNull) { - tdResetDataCols(target); - ASSERT(limit1 <= src1->numOfRows && limit2 <= src2->numOfRows); - - while (target->numOfRows < tRows) { - if (*iter1 >= limit1 && *iter2 >= limit2) break; - - TSKEY key1 = (*iter1 >= limit1) ? INT64_MAX : dataColsKeyAt(src1, *iter1); - TKEY tkey1 = (*iter1 >= limit1) ? TKEY_NULL : dataColsTKeyAt(src1, *iter1); - TSKEY key2 = (*iter2 >= limit2) ? INT64_MAX : dataColsKeyAt(src2, *iter2); - TKEY tkey2 = (*iter2 >= limit2) ? TKEY_NULL : dataColsTKeyAt(src2, *iter2); - - ASSERT(tkey1 == TKEY_NULL || (!TKEY_IS_DELETED(tkey1))); - - if (key1 < key2) { - for (int i = 0; i < src1->numOfCols; i++) { - ASSERT(target->cols[i].type == src1->cols[i].type); - if (src1->cols[i].len > 0 || target->cols[i].len > 0) { - dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src1->cols + i, *iter1), target->numOfRows, - target->maxPoints); - } - } - - target->numOfRows++; - (*iter1)++; - } else if (key1 >= key2) { - if ((key1 > key2) || (key1 == key2 && !TKEY_IS_DELETED(tkey2))) { - for (int i = 0; i < src2->numOfCols; i++) { - ASSERT(target->cols[i].type == src2->cols[i].type); - if (src2->cols[i].len > 0 && !isNull(src2->cols[i].pData, src2->cols[i].type)) { - dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfRows, - target->maxPoints); - } else if(!forceSetNull && key1 == key2 && src1->cols[i].len > 0) { - dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src1->cols + i, *iter1), target->numOfRows, - target->maxPoints); - } else if(target->cols[i].len > 0) { - dataColSetNullAt(&target->cols[i], target->numOfRows); - } - } - target->numOfRows++; - } - - (*iter2)++; - if (key1 == key2) (*iter1)++; - } - - ASSERT(target->numOfRows <= target->maxPoints); - } -} -#endif SKVRow tdKVRowDup(SKVRow row) { SKVRow trow = taosMemoryMalloc(kvRowLen(row)); @@ -859,98 +504,3 @@ SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder) { return row; } -#if 0 -SMemRow mergeTwoMemRows(void *buffer, SMemRow row1, SMemRow row2, STSchema *pSchema1, STSchema *pSchema2) { -#if 0 - ASSERT(memRowKey(row1) == memRowKey(row2)); - ASSERT(schemaVersion(pSchema1) == memRowVersion(row1)); - ASSERT(schemaVersion(pSchema2) == memRowVersion(row2)); - ASSERT(schemaVersion(pSchema1) >= schemaVersion(pSchema2)); -#endif - - SArray *stashRow = taosArrayInit(pSchema1->numOfCols, sizeof(SColInfo)); - if (stashRow == NULL) { - return NULL; - } - - SMemRow pRow = buffer; - SDataRow dataRow = memRowDataBody(pRow); - memRowSetType(pRow, SMEM_ROW_DATA); - dataRowSetVersion(dataRow, schemaVersion(pSchema1)); // use latest schema version - dataRowSetLen(dataRow, (TDRowLenT)(TD_DATA_ROW_HEAD_SIZE + pSchema1->flen)); - - TDRowLenT dataLen = 0, kvLen = TD_MEM_ROW_KV_HEAD_SIZE; - - int32_t i = 0; // row1 - int32_t j = 0; // row2 - int32_t nCols1 = schemaNCols(pSchema1); - int32_t nCols2 = schemaNCols(pSchema2); - SColInfo colInfo = {0}; - int32_t kvIdx1 = 0, kvIdx2 = 0; - - while (i < nCols1) { - STColumn *pCol = schemaColAt(pSchema1, i); - void * val1 = tdGetMemRowDataOfColEx(row1, pCol->colId, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset, &kvIdx1); - // if val1 != NULL, use val1; - if (val1 != NULL && !isNull(val1, pCol->type)) { - tdAppendColVal(dataRow, val1, pCol->type, pCol->offset); - kvLen += tdGetColAppendLen(SMEM_ROW_KV, val1, pCol->type); - setSColInfo(&colInfo, pCol->colId, pCol->type, val1); - taosArrayPush(stashRow, &colInfo); - ++i; // next col - continue; - } - - void *val2 = NULL; - while (j < nCols2) { - STColumn *tCol = schemaColAt(pSchema2, j); - if (tCol->colId < pCol->colId) { - ++j; - continue; - } - if (tCol->colId == pCol->colId) { - val2 = tdGetMemRowDataOfColEx(row2, tCol->colId, tCol->type, TD_DATA_ROW_HEAD_SIZE + tCol->offset, &kvIdx2); - } else if (tCol->colId > pCol->colId) { - // set NULL - } - break; - } // end of while(jtype); - } - tdAppendColVal(dataRow, val2, pCol->type, pCol->offset); - if (!isNull(val2, pCol->type)) { - kvLen += tdGetColAppendLen(SMEM_ROW_KV, val2, pCol->type); - setSColInfo(&colInfo, pCol->colId, pCol->type, val2); - taosArrayPush(stashRow, &colInfo); - } - - ++i; // next col - } - - dataLen = memRowTLen(pRow); - - if (kvLen < dataLen) { - // scan stashRow and generate SKVRow - memset(buffer, 0, sizeof(dataLen)); - SMemRow tRow = buffer; - memRowSetType(tRow, SMEM_ROW_KV); - SKVRow kvRow = (SKVRow)memRowKvBody(tRow); - int16_t nKvNCols = (int16_t) taosArrayGetSize(stashRow); - kvRowSetLen(kvRow, (TDRowLenT)(TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nKvNCols)); - kvRowSetNCols(kvRow, nKvNCols); - memRowSetKvVersion(tRow, pSchema1->version); - - int32_t toffset = 0; - int16_t k; - for (k = 0; k < nKvNCols; ++k) { - SColInfo *pColInfo = taosArrayGet(stashRow, k); - tdAppendKvColVal(kvRow, pColInfo->colVal, true, pColInfo->colId, pColInfo->colType, toffset); - toffset += sizeof(SColIdx); - } - ASSERT(kvLen == memRowTLen(tRow)); - } - taosArrayDestroy(stashRow); - return buffer; -} -#endif diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 6c0df33d05..1315963090 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -211,7 +211,7 @@ int tsdbCommit(STsdb *pRepo) { void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn) { STsdbKeepCfg *pCfg = REPO_KEEP_CFG(pRepo); - TSKEY minKey, midKey, maxKey, now; + TSKEY minKey, midKey, maxKey, now; now = taosGetTimestamp(pCfg->precision); minKey = now - pCfg->keep2 * tsTickPerDay[pCfg->precision]; @@ -1386,34 +1386,7 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt tSkipListIterNext(pCommitIter->pIter); } else { -#if 0 - if (update != TD_ROW_OVERWRITE_UPDATE) { - // copy disk data - for (int i = 0; i < pDataCols->numOfCols; ++i) { - // TODO: dataColAppendVal may fail - SCellVal sVal = {0}; - if (tdGetColDataOfRow(&sVal, pDataCols->cols + i, *iter, pDataCols->bitmapMode) < 0) { - TASSERT(0); - } - tdAppendValToDataCol(pTarget->cols + i, sVal.valType, sVal.val, pTarget->numOfRows, pTarget->maxPoints, pTarget->bitmapMode); - } - - if (update == TD_ROW_DISCARD_UPDATE) pTarget->numOfRows++; - } - if (update != TD_ROW_DISCARD_UPDATE) { - // copy mem data - if (pSchema == NULL || schemaVersion(pSchema) != TD_ROW_SVER(row)) { - pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, TD_ROW_SVER(row)); - ASSERT(pSchema != NULL); - } - - tdAppendSTSRowToDataCol(row, pSchema, pTarget, update == TD_ROW_OVERWRITE_UPDATE); - } - ++(*iter); - tSkipListIterNext(pCommitIter->pIter); -#endif - - if(lastKey != key1) { + if (lastKey != key1) { lastKey = key1; ++pTarget->numOfRows; } @@ -1484,29 +1457,4 @@ static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *p } return false; -} - -// int tsdbApplyRtn(STsdbRepo *pRepo) { -// SRtn rtn; -// SFSIter fsiter; -// STsdbFS * pfs = REPO_FS(pRepo); -// SDFileSet *pSet; - -// // Get retention snapshot -// tsdbGetRtnSnap(pRepo, &rtn); - -// tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD); -// while ((pSet = tsdbFSIterNext(&fsiter))) { -// if (pSet->fid < rtn.minFid) { -// tsdbInfo("vgId:%d FSET %d at level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid, -// TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); -// continue; -// } - -// if (tsdbApplyRtnOnFSet(pRepo, pSet, &rtn) < 0) { -// return -1; -// } -// } - -// return 0; -// } +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index e9dc98d102..3168ff53f6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -21,6 +21,8 @@ typedef struct SMemSkipList SMemSkipList; typedef struct SMemSkipListNode SMemSkipListNode; typedef struct SMemSkipListCurosr SMemSkipListCurosr; +#define SL_MAX_LEVEL 5 + struct SMemTable { STsdb *pTsdb; TSKEY minKey; @@ -61,7 +63,7 @@ struct SMemData { struct SMemSkipListCurosr { SMemSkipList *pSl; - SMemSkipListNode *pNodeC; + SMemSkipListNode *pNodes[SL_MAX_LEVEL]; }; typedef struct { @@ -70,8 +72,6 @@ typedef struct { const STSRow *pRow; } STsdbRow; -#define SL_MAX_LEVEL 15 - #define HASH_BUCKET(SUID, UID, NBUCKET) (TABS((SUID) + (UID)) % (NBUCKET)) #define SL_NODE_SIZE(l) (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l)*2) @@ -93,6 +93,8 @@ static void tsdbMemSkipListCursorDestroy(SMemSkipListCurosr *pSlc); static void tsdbMemSkipListCursorInit(SMemSkipListCurosr *pSlc, SMemSkipList *pSl); static void tsdbMemSkipListCursorPut(SMemSkipListCurosr *pSlc, SMemSkipListNode *pNode); static int32_t tsdbMemSkipListCursorMoveTo(SMemSkipListCurosr *pSlc, int64_t version, TSKEY ts, int32_t flags); +static void tsdbMemSkipListCursorMoveToFirst(SMemSkipListCurosr *pSlc); +static void tsdbMemSkipListCursorMoveToLast(SMemSkipListCurosr *pSlc); static int32_t tsdbMemSkipListCursorMoveToNext(SMemSkipListCurosr *pSlc); static int32_t tsdbMemSkipListCursorMoveToPrev(SMemSkipListCurosr *pSlc); static SMemSkipListNode *tsdbMemSkipListNodeCreate(SVBufPool *pPool, SMemSkipList *pSl, const STsdbRow *pTRow); @@ -326,14 +328,26 @@ static int32_t tsdbMemSkipListCursorMoveTo(SMemSkipListCurosr *pSlc, int64_t ver return 0; } -static int32_t tsdbMemSkipListCursorMoveToFirst(SMemSkipListCurosr *pSlc) { - // TODO - return 0; +static void tsdbMemSkipListCursorMoveToFirst(SMemSkipListCurosr *pSlc) { + SMemSkipList *pSl = pSlc->pSl; + SMemSkipListNode *pHead = SL_HEAD_NODE(pSl); + + for (int8_t iLevel = 0; iLevel < pSl->maxLevel; iLevel++) { + pSlc->pNodes[iLevel] = pHead; + } + + tsdbMemSkipListCursorMoveToNext(pSlc); } -static int32_t tsdbMemSkipListCursorMoveToLast(SMemSkipListCurosr *pSlc) { - // TODO - return 0; +static void tsdbMemSkipListCursorMoveToLast(SMemSkipListCurosr *pSlc) { + SMemSkipList *pSl = pSlc->pSl; + SMemSkipListNode *pTail = SL_TAIL_NODE(pSl); + + for (int8_t iLevel = 0; iLevel < pSl->maxLevel; iLevel++) { + pSlc->pNodes[iLevel] = pTail; + } + + tsdbMemSkipListCursorMoveToPrev(pSlc); } static int32_t tsdbMemSkipListCursorMoveToNext(SMemSkipListCurosr *pSlc) { From 79570c93a357bcdffe2c5c0b499e39d6be3344b6 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 9 May 2022 19:37:16 +0800 Subject: [PATCH 14/29] feat(query): add leastsquares function --- include/libs/function/functionMgt.h | 2 +- source/libs/function/inc/builtinsimpl.h | 6 + source/libs/function/src/builtins.c | 28 ++++ source/libs/function/src/builtinsimpl.c | 184 +++++++++++++++++++++++- 4 files changed, 217 insertions(+), 3 deletions(-) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 4a37283ee5..26bf566808 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -31,13 +31,13 @@ typedef enum EFunctionType { FUNCTION_TYPE_ELAPSED, FUNCTION_TYPE_IRATE, FUNCTION_TYPE_LAST_ROW, - FUNCTION_TYPE_LEASTSQUARES, FUNCTION_TYPE_MAX, FUNCTION_TYPE_MIN, FUNCTION_TYPE_MODE, FUNCTION_TYPE_PERCENTILE, FUNCTION_TYPE_SPREAD, FUNCTION_TYPE_STDDEV, + FUNCTION_TYPE_LEASTSQUARES, FUNCTION_TYPE_SUM, FUNCTION_TYPE_TWA, FUNCTION_TYPE_HISTOGRAM, diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 1f2ad0797d..ae989f3280 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -55,6 +55,12 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx); int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t stddevInvertFunction(SqlFunctionCtx* pCtx); +bool getLeastSQRFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); +bool leastSQRFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); +int32_t leastSQRFunction(SqlFunctionCtx* pCtx); +int32_t leastSQRFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); +int32_t leastSQRInvertFunction(SqlFunctionCtx* pCtx); + bool getPercentileFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool percentileFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); int32_t percentileFunction(SqlFunctionCtx *pCtx); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 5aa1b63c79..0eb014646b 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -225,6 +225,23 @@ static int32_t translateSpread(SFunctionNode* pFunc, char* pErrBuf, int32_t len) return TSDB_CODE_SUCCESS; } +static int32_t translateLeastSQR(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); + if (3 != numOfParams) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } + + for (int32_t i = 0; i < numOfParams; ++i) { + uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, i))->resType.type; + if (!IS_NUMERIC_TYPE(colType)) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + } + + pFunc->node.resType = (SDataType) { .bytes = 64, .type = TSDB_DATA_TYPE_BINARY }; + return TSDB_CODE_SUCCESS; +} + static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { if (4 != LIST_LENGTH(pFunc->pParameterList)) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); @@ -535,6 +552,17 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .finalizeFunc = stddevFinalize, .invertFunc = stddevInvertFunction }, + { + .name = "leastsquares", + .type = FUNCTION_TYPE_LEASTSQUARES, + .classification = FUNC_MGT_AGG_FUNC, + .translateFunc = translateLeastSQR, + .getEnvFunc = getLeastSQRFuncEnv, + .initFunc = leastSQRFunctionSetup, + .processFunc = leastSQRFunction, + .finalizeFunc = leastSQRFinalize, + .invertFunc = leastSQRInvertFunction + }, { .name = "avg", .type = FUNCTION_TYPE_AVG, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 9c1601b61a..deda98ed0e 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -63,6 +63,13 @@ typedef struct SStddevRes { }; } SStddevRes; +typedef struct SLeastSQRInfo { + double matrix[2][3]; + double startVal; + double stepVal; + int64_t num; +} SLeastSQRInfo; + typedef struct SPercentileInfo { double result; tMemBucket* pMemBucket; @@ -1112,6 +1119,179 @@ int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return functionFinalize(pCtx, pBlock); } +bool getLeastSQRFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { + pEnv->calcMemSize = sizeof(SLeastSQRInfo); + return true; +} + +bool leastSQRFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { + if (!functionSetup(pCtx, pResultInfo)) { + return false; + } + + SLeastSQRInfo* pInfo = GET_ROWCELL_INTERBUF(pResultInfo); + + pInfo->startVal = pCtx->param[1].param.d; + pInfo->stepVal = pCtx->param[2].param.d; + return true; +} + +#define LEASTSQR_CAL(p, x, y, index, step) \ + do { \ + (p)[0][0] += (double)(x) * (x); \ + (p)[0][1] += (double)(x); \ + (p)[0][2] += (double)(x) * (y)[index]; \ + (p)[1][2] += (y)[index]; \ + (x) += step; \ + } while (0) + +int32_t leastSQRFunction(SqlFunctionCtx* pCtx) { + int32_t numOfElem = 0; + + SInputColumnInfoData* pInput = &pCtx->input; + int32_t type = pInput->pData[0]->info.type; + + SLeastSQRInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + + SColumnInfoData* pCol = pInput->pData[0]; + + double(*param)[3] = pInfo->matrix; + double x = pInfo->startVal; + + int32_t start = pInput->startRowIndex; + int32_t numOfRows = pInput->numOfRows; + + switch (type) { + case TSDB_DATA_TYPE_TINYINT: { + int8_t* plist = (int8_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + numOfElem++; + LEASTSQR_CAL(param, x, plist, i, pInfo->stepVal); + + break; + } + } + case TSDB_DATA_TYPE_SMALLINT: { + int16_t* plist = (int16_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem++; + LEASTSQR_CAL(param, x, plist, i, pInfo->stepVal); + } + break; + } + + case TSDB_DATA_TYPE_INT: { + int32_t* plist = (int32_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem++; + LEASTSQR_CAL(param, x, plist, i, pInfo->stepVal); + } + + break; + } + + case TSDB_DATA_TYPE_BIGINT: { + int64_t* plist = (int64_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem++; + LEASTSQR_CAL(param, x, plist, i, pInfo->stepVal); + } + break; + } + + case TSDB_DATA_TYPE_FLOAT: { + float* plist = (float*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem++; + LEASTSQR_CAL(param, x, plist, i, pInfo->stepVal); + } + break; + } + + case TSDB_DATA_TYPE_DOUBLE: { + double* plist = (double*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem++; + LEASTSQR_CAL(param, x, plist, i, pInfo->stepVal); + } + break; + } + + default: + break; + } + + pInfo->startVal = x; + pInfo->num += numOfElem; + + SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1); + + return TSDB_CODE_SUCCESS; +} + +int32_t leastSQRFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + SLeastSQRInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + int32_t slotId = pCtx->pExpr->base.resSchema.slotId; + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + + int32_t currentRow = pBlock->info.rows; + + if (pInfo->num = 0) { + return 0; + } + + double(*param)[3] = pInfo->matrix; + + param[1][1] = (double)pInfo->num; + param[1][0] = param[0][1]; + + param[0][0] -= param[1][0] * (param[0][1] / param[1][1]); + param[0][2] -= param[1][2] * (param[0][1] / param[1][1]); + param[0][1] = 0; + param[1][2] -= param[0][2] * (param[1][0] / param[0][0]); + param[1][0] = 0; + param[0][2] /= param[0][0]; + + param[1][2] /= param[1][1]; + + char buf[64] = {0}; + size_t len = snprintf(varDataVal(buf), sizeof(buf) - VARSTR_HEADER_SIZE, "{slop:%.6lf, intercept:%.6lf}", param[0][2], param[1][2]); + varDataSetLen(buf, len); + + colDataAppend(pCol, currentRow, buf, false); + + return pResInfo->numOfRes; +} + +int32_t leastSQRInvertFunction(SqlFunctionCtx* pCtx) { + //TODO + return TSDB_CODE_SUCCESS; +} + bool getPercentileFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { pEnv->calcMemSize = sizeof(SPercentileInfo); return true; @@ -2184,10 +2364,10 @@ int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t len; char buf[512] = {0}; if (!pInfo->normalized) { - len = sprintf(buf + VARSTR_HEADER_SIZE, "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%"PRId64"}", + len = sprintf(varDataVal(buf), "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%"PRId64"}", pInfo->bins[i].lower, pInfo->bins[i].upper, pInfo->bins[i].count); } else { - len = sprintf(buf + VARSTR_HEADER_SIZE, "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%lf}", + len = sprintf(varDataVal(buf), "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%lf}", pInfo->bins[i].lower, pInfo->bins[i].upper, pInfo->bins[i].percentage); } varDataSetLen(buf, len); From 1157154d1e1233f505edb08a89bb7d486fa66021 Mon Sep 17 00:00:00 2001 From: dapan Date: Mon, 9 May 2022 20:03:09 +0800 Subject: [PATCH 15/29] stmt test --- source/client/src/clientStmt.c | 6 ++++-- source/libs/parser/src/parInsertData.c | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 737282d214..c804213d89 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -465,10 +465,12 @@ int stmtSetTbTags(TAOS_STMT *stmt, TAOS_MULTI_BIND *tags) { STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTAGS)); - if (pStmt->bInfo.needParse) { - STMT_ERR_RET(stmtParseSql(pStmt)); + if (!pStmt->bInfo.needParse) { + return TSDB_CODE_SUCCESS; } + STMT_ERR_RET(stmtParseSql(pStmt)); + STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid)); if (NULL == pDataBlock) { tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid); diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index 5fa2961474..c074334722 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -601,6 +601,7 @@ int32_t qResetStmtDataBlock(void* block, bool keepBuf) { pBlock->numOfTables = 1; pBlock->nAllocSize = TSDB_PAYLOAD_SIZE; pBlock->headerSize = pBlock->size; + pBlock->createTbReqLen = 0; memset(&pBlock->rowBuilder, 0, sizeof(pBlock->rowBuilder)); From cfa77a2d398ca370fec8d856b8dd87e97da4dab5 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 9 May 2022 19:37:16 +0800 Subject: [PATCH 16/29] feat(query): add leastsquares function --- source/libs/function/src/builtinsimpl.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index deda98ed0e..75efb533ad 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -1131,8 +1131,10 @@ bool leastSQRFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInf SLeastSQRInfo* pInfo = GET_ROWCELL_INTERBUF(pResultInfo); - pInfo->startVal = pCtx->param[1].param.d; - pInfo->stepVal = pCtx->param[2].param.d; + pInfo->startVal = IS_FLOAT_TYPE(pCtx->param[1].param.nType) ? pCtx->param[1].param.d : + (double)pCtx->param[1].param.i; + pInfo->stepVal = IS_FLOAT_TYPE(pCtx->param[1].param.nType) ? pCtx->param[2].param.d : + (double)pCtx->param[1].param.i; return true; } @@ -1260,7 +1262,7 @@ int32_t leastSQRFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t currentRow = pBlock->info.rows; - if (pInfo->num = 0) { + if (0 == pInfo->num) { return 0; } From 3cc90115792c65cd98dc6807ba2b77b61fa481de Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 9 May 2022 20:35:59 +0800 Subject: [PATCH 17/29] fix case --- source/dnode/mnode/impl/test/user/user.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/test/user/user.cpp b/source/dnode/mnode/impl/test/user/user.cpp index 1e03d8ff4a..3c15bc3492 100644 --- a/source/dnode/mnode/impl/test/user/user.cpp +++ b/source/dnode/mnode/impl/test/user/user.cpp @@ -367,8 +367,8 @@ TEST_F(MndTestUser, 03_Alter_User) { EXPECT_EQ(numOfReadDbs, 1); EXPECT_EQ(numOfWriteDbs, 0); - char* dbname = (char*)taosHashGet(authRsp.readDbs, "1.d2", 5); - EXPECT_STREQ(dbname, "1.d2"); + char* dbname = (char*)taosHashGet(authRsp.readDbs, "1.d2", 4); + EXPECT_TRUE(dbname != NULL); taosHashCleanup(authRsp.readDbs); taosHashCleanup(authRsp.writeDbs); From 76a9ef186efde3a66cd194abef6f558a1bd48bac Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 9 May 2022 13:10:16 +0000 Subject: [PATCH 18/29] refact row apis --- include/common/trow.h | 1159 ++------------------------------------ source/common/src/trow.c | 934 ++++++++++++++++++++++++++++++ 2 files changed, 975 insertions(+), 1118 deletions(-) diff --git a/include/common/trow.h b/include/common/trow.h index ebce0b085d..eed15689c0 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -95,8 +95,6 @@ static FORCE_INLINE bool tdValIsNull(TDRowValT valType, const void *val, int32_t #endif } -typedef void *SRow; - typedef struct { TDRowValT valType; void *val; @@ -228,24 +226,19 @@ static FORCE_INLINE void *tdKVRowColVal(STSRow *pRow, SKvRowIdx *pIdx) { return void tdMergeBitmap(uint8_t *srcBitmap, int32_t nBits, uint8_t *dstBitmap); static FORCE_INLINE void tdRowCopy(void *dst, STSRow *row) { memcpy(dst, row, TD_ROW_LEN(row)); } -static FORCE_INLINE int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType); static FORCE_INLINE int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT valType); static FORCE_INLINE int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int8_t bitmapMode); int32_t tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDRowValT valType, int8_t bitmapMode); -static FORCE_INLINE int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType); -static FORCE_INLINE int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pValType); static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode); -static FORCE_INLINE bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx, int8_t bitmapMode); bool tdIsBitmapBlkNorm(const void *pBitmap, int32_t numOfBits, int8_t bitmapMode); int32_t tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int32_t numOfRows, int32_t maxPoints, int8_t bitmapMode, bool isMerge); -static FORCE_INLINE int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, - bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset); -static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, - bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset, - col_id_t colId); -int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols, bool isMerge); +int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols, bool isMerge); + +int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pValType); +int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType); +int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType); /** * @brief @@ -264,257 +257,11 @@ static FORCE_INLINE void *tdGetBitmapAddrKv(STSRow *pRow, col_id_t nKvCols) { // The primary TS key is stored separatedly and is Norm value, thus should minus 1 firstly return POINTER_SHIFT(TD_ROW_COL_IDX(pRow), (--nKvCols) * sizeof(SKvRowIdx)); } -static FORCE_INLINE void *tdGetBitmapAddr(STSRow *pRow, uint8_t rowType, uint32_t flen, col_id_t nKvCols) { -#ifdef TD_SUPPORT_BITMAP - switch (rowType) { - case TD_ROW_TP: - return tdGetBitmapAddrTp(pRow, flen); - case TD_ROW_KV: - return tdGetBitmapAddrKv(pRow, nKvCols); - default: - break; - } -#endif - return NULL; -} - -static FORCE_INLINE int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int8_t bitmapMode) { - switch (bitmapMode) { - case 0: - tdSetBitmapValTypeII(pBitmap, colIdx, valType); - break; - case -1: - case 1: - tdSetBitmapValTypeI(pBitmap, colIdx, valType); - break; - default: - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return TSDB_CODE_FAILED; - } - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Use 2 bits at default - * - * @param pBitmap - * @param colIdx The relative index of colId, may have minus value as parameter. - * @param valType - * @return FORCE_INLINE - */ -static FORCE_INLINE int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT valType) { - if (!pBitmap || colIdx < 0) { - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - int16_t nBytes = colIdx / TD_VTYPE_PARTS; - int16_t nOffset = colIdx & TD_VTYPE_OPTR; - char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); - // use literal value directly and not use formula to simplify the codes - switch (nOffset) { - case 0: - *pDestByte = ((*pDestByte) & 0x3F) | (valType << 6); - // set the value and clear other partitions for offset 0 - // *pDestByte |= (valType << 6); - break; - case 1: - *pDestByte = ((*pDestByte) & 0xCF) | (valType << 4); - // *pDestByte |= (valType << 4); - break; - case 2: - *pDestByte = ((*pDestByte) & 0xF3) | (valType << 2); - // *pDestByte |= (valType << 2); - break; - case 3: - *pDestByte = ((*pDestByte) & 0xFC) | valType; - // *pDestByte |= (valType); - break; - default: - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - return TSDB_CODE_SUCCESS; -} - -static FORCE_INLINE bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx, int8_t bitmapMode) { - TDRowValT valType = 0; - tdGetBitmapValType(pBitmap, idx, &valType, bitmapMode); - if (tdValTypeIsNorm(valType)) { - return true; - } - return false; -} - -static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, - int8_t bitmapMode) { - switch (bitmapMode) { - case 0: - tdGetBitmapValTypeII(pBitmap, colIdx, pValType); - break; - case -1: - case 1: - tdGetBitmapValTypeI(pBitmap, colIdx, pValType); - break; - default: - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return TSDB_CODE_FAILED; - } - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Use 2 bits at default - * - * @param pBitmap - * @param colIdx The relative index of colId, may have minus value as parameter. - * @param pValType - * @return FORCE_INLINE - */ -static FORCE_INLINE int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) { - if (!pBitmap || colIdx < 0) { - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - int16_t nBytes = colIdx / TD_VTYPE_PARTS; - int16_t nOffset = colIdx & TD_VTYPE_OPTR; - char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); - // use literal value directly and not use formula to simplify the codes - switch (nOffset) { - case 0: - *pValType = (((*pDestByte) & 0xC0) >> 6); - break; - case 1: - *pValType = (((*pDestByte) & 0x30) >> 4); - break; - case 2: - *pValType = (((*pDestByte) & 0x0C) >> 2); - break; - case 3: - *pValType = ((*pDestByte) & 0x03); - break; - default: - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - return TSDB_CODE_SUCCESS; -} - -/** - * @brief - * - * @param pBitmap - * @param colIdx The relative index of colId, may have minus value as parameter. - * @param valType - * @return FORCE_INLINE - */ -static FORCE_INLINE int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType) { - if (!pBitmap || colIdx < 0) { - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - int16_t nBytes = colIdx / TD_VTYPE_PARTS_I; - int16_t nOffset = colIdx & TD_VTYPE_OPTR_I; - char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); - // use literal value directly and not use formula to simplify the codes - switch (nOffset) { - case 0: - *pDestByte = ((*pDestByte) & 0x7F) | (valType << 7); - // set the value and clear other partitions for offset 0 - // *pDestByte |= (valType << 7); - break; - case 1: - *pDestByte = ((*pDestByte) & 0xBF) | (valType << 6); - // *pDestByte |= (valType << 6); - break; - case 2: - *pDestByte = ((*pDestByte) & 0xDF) | (valType << 5); - // *pDestByte |= (valType << 5); - break; - case 3: - *pDestByte = ((*pDestByte) & 0xEF) | (valType << 4); - // *pDestByte |= (valType << 4); - break; - case 4: - *pDestByte = ((*pDestByte) & 0xF7) | (valType << 3); - // *pDestByte |= (valType << 3); - break; - case 5: - *pDestByte = ((*pDestByte) & 0xFB) | (valType << 2); - // *pDestByte |= (valType << 2); - break; - case 6: - *pDestByte = ((*pDestByte) & 0xFD) | (valType << 1); - // *pDestByte |= (valType << 1); - break; - case 7: - *pDestByte = ((*pDestByte) & 0xFE) | valType; - // *pDestByte |= (valType); - break; - default: - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - return TSDB_CODE_SUCCESS; -} - -/** - * @brief - * - * @param pBitmap - * @param colIdx The relative index of colId, may have minus value as parameter. - * @param pValType - * @return FORCE_INLINE - */ -static FORCE_INLINE int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) { - if (!pBitmap || colIdx < 0) { - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - int16_t nBytes = colIdx / TD_VTYPE_PARTS_I; - int16_t nOffset = colIdx & TD_VTYPE_OPTR_I; - char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); - // use literal value directly and not use formula to simplify the codes - switch (nOffset) { - case 0: - *pValType = (((*pDestByte) & 0x80) >> 7); - break; - case 1: - *pValType = (((*pDestByte) & 0x40) >> 6); - break; - case 2: - *pValType = (((*pDestByte) & 0x20) >> 5); - break; - case 3: - *pValType = (((*pDestByte) & 0x10) >> 4); - break; - case 4: - *pValType = (((*pDestByte) & 0x08) >> 3); - break; - case 5: - *pValType = (((*pDestByte) & 0x04) >> 2); - break; - case 6: - *pValType = (((*pDestByte) & 0x02) >> 1); - break; - case 7: - *pValType = ((*pDestByte) & 0x01); - break; - default: - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - return TSDB_CODE_SUCCESS; -} +void *tdGetBitmapAddr(STSRow *pRow, uint8_t rowType, uint32_t flen, col_id_t nKvCols); +int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int8_t bitmapMode); +int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT valType); +bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx, int8_t bitmapMode); +int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode); // ----------------- Tuple row structure(STpRow) /* @@ -539,482 +286,28 @@ static FORCE_INLINE int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t col * */ -/** - * @brief - * - * @param pBuilder - * @param sver schema version - * @return FORCE_INLINE - */ static FORCE_INLINE void tdSRowInit(SRowBuilder *pBuilder, int16_t sver) { pBuilder->rowType = TD_ROW_TP; // default STpRow pBuilder->sver = sver; } - -/** - * @brief Not recommended to use - * - * @param pBuilder - * @param rowType - * @return FORCE_INLINE - */ -static FORCE_INLINE void tdSRowSetRowType(SRowBuilder *pBuilder, int8_t rowType) { pBuilder->rowType = rowType; } - -/** - * @brief - * - * @param pBuilder - * @param nCols - * @param nBoundCols use -1 if not available - * @param flen - * @return FORCE_INLINE - */ -static FORCE_INLINE int32_t tdSRowSetInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen) { - pBuilder->flen = flen; - pBuilder->nCols = nCols; - pBuilder->nBoundCols = nBoundCols; - if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) { - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } -#ifdef TD_SUPPORT_BITMAP - // the primary TS key is stored separatedly - pBuilder->nBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nCols - 1); - if (nBoundCols > 0) { - pBuilder->nBoundBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nBoundCols - 1); - } else { - pBuilder->nBoundBitmaps = 0; - } -#else - pBuilder->nBitmaps = 0; - pBuilder->nBoundBitmaps = 0; -#endif - return TSDB_CODE_SUCCESS; -} - -/** - * @brief - * - * @param pBuilder - * @param nCols - * @param nBoundCols use -1 if not available - * @param flen - * @return FORCE_INLINE - */ -static FORCE_INLINE int32_t tdSRowSetTpInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t flen) { - pBuilder->flen = flen; - pBuilder->nCols = nCols; - if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) { - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } -#ifdef TD_SUPPORT_BITMAP - // the primary TS key is stored separatedly - pBuilder->nBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nCols - 1); -#else - pBuilder->nBitmaps = 0; - pBuilder->nBoundBitmaps = 0; -#endif - return TSDB_CODE_SUCCESS; -} - -/** - * @brief To judge row type: STpRow/SKvRow - * - * @param pBuilder - * @param nCols - * @param nBoundCols - * @param flen - * @param allNullLen use -1 if not available - * @param boundNullLen use -1 if not available - * @return FORCE_INLINE - */ -static FORCE_INLINE int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, - int32_t flen, int32_t allNullLen, int32_t boundNullLen) { - if ((boundNullLen > 0) && (allNullLen > 0) && (nBoundCols > 0)) { - uint32_t tpLen = allNullLen; - uint32_t kvLen = sizeof(col_id_t) + sizeof(SKvRowIdx) * nBoundCols + boundNullLen; - if (isSelectKVRow(kvLen, tpLen)) { - pBuilder->rowType = TD_ROW_KV; - } else { - pBuilder->rowType = TD_ROW_TP; - } - - } else { - pBuilder->rowType = TD_ROW_TP; - } - - pBuilder->flen = flen; - pBuilder->nCols = nCols; - pBuilder->nBoundCols = nBoundCols; - if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) { - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } -#ifdef TD_SUPPORT_BITMAP - // the primary TS key is stored separatedly - pBuilder->nBitmaps = (col_id_t)TD_BITMAP_BYTES(pBuilder->nCols - 1); - if (nBoundCols > 0) { - pBuilder->nBoundBitmaps = (col_id_t)TD_BITMAP_BYTES(pBuilder->nBoundCols - 1); - } else { - pBuilder->nBoundBitmaps = 0; - } -#else - pBuilder->nBitmaps = 0; - pBuilder->nBoundBitmaps = 0; -#endif - return TSDB_CODE_SUCCESS; -} - -/** - * @brief The invoker is responsible for memory alloc/dealloc. - * - * @param pBuilder - * @param pBuf Output buffer of STSRow - */ -static int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { - pBuilder->pBuf = (STSRow *)pBuf; - if (!pBuilder->pBuf) { - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - - TD_ROW_SET_INFO(pBuilder->pBuf, 0); - TD_ROW_SET_TYPE(pBuilder->pBuf, pBuilder->rowType); - - TASSERT(pBuilder->nBitmaps > 0 && pBuilder->flen > 0); - - uint32_t len = 0; - switch (pBuilder->rowType) { - case TD_ROW_TP: -#ifdef TD_SUPPORT_BITMAP - pBuilder->pBitmap = tdGetBitmapAddrTp(pBuilder->pBuf, pBuilder->flen); - memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBitmaps); -#endif - // the primary TS key is stored separatedly - len = TD_ROW_HEAD_LEN + pBuilder->flen - sizeof(TSKEY) + pBuilder->nBitmaps; - TD_ROW_SET_LEN(pBuilder->pBuf, len); - TD_ROW_SET_SVER(pBuilder->pBuf, pBuilder->sver); - break; - case TD_ROW_KV: -#ifdef TD_SUPPORT_BITMAP - pBuilder->pBitmap = tdGetBitmapAddrKv(pBuilder->pBuf, pBuilder->nBoundCols); - memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBoundBitmaps); -#endif - len = TD_ROW_HEAD_LEN + TD_ROW_NCOLS_LEN + (pBuilder->nBoundCols - 1) * sizeof(SKvRowIdx) + - pBuilder->nBoundBitmaps; // add - TD_ROW_SET_LEN(pBuilder->pBuf, len); - TD_ROW_SET_SVER(pBuilder->pBuf, pBuilder->sver); - TD_ROW_SET_NCOLS(pBuilder->pBuf, pBuilder->nBoundCols); - break; - default: - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - - return TSDB_CODE_SUCCESS; -} - -/** - * @brief The invoker is responsible for memory alloc/dealloc. - * - * @param pBuilder - * @param pBuf Output buffer of STSRow - */ -static int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf) { - pBuilder->pBuf = (STSRow *)pBuf; - if (!pBuilder->pBuf) { - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - - TASSERT(pBuilder->nBitmaps > 0 && pBuilder->flen > 0); - - uint32_t len = 0; - switch (pBuilder->rowType) { - case TD_ROW_TP: -#ifdef TD_SUPPORT_BITMAP - pBuilder->pBitmap = tdGetBitmapAddrTp(pBuilder->pBuf, pBuilder->flen); -#endif - break; - case TD_ROW_KV: -#ifdef TD_SUPPORT_BITMAP - pBuilder->pBitmap = tdGetBitmapAddrKv(pBuilder->pBuf, pBuilder->nBoundCols); -#endif - break; - default: - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - return TSDB_CODE_SUCCESS; -} - -/** - * @brief 由调用方管理存储空间的分配及释放，一次输入多个参数 - * - * @param pBuilder - * @param pBuf - * @param allNullLen - * @param boundNullLen - * @param nCols - * @param nBoundCols - * @param flen - * @return FORCE_INLINE - */ -static FORCE_INLINE int32_t tdSRowInitEx(SRowBuilder *pBuilder, void *pBuf, uint32_t allNullLen, uint32_t boundNullLen, - int32_t nCols, int32_t nBoundCols, int32_t flen) { - if (tdSRowSetExtendedInfo(pBuilder, allNullLen, boundNullLen, nCols, nBoundCols, flen) < 0) { - return terrno; - } - return tdSRowResetBuf(pBuilder, pBuf); -} - -/** - * @brief - * - * @param pBuilder - */ -static FORCE_INLINE void tdSRowReset(SRowBuilder *pBuilder) { - pBuilder->rowType = TD_ROW_TP; - pBuilder->pBuf = NULL; - pBuilder->nBoundCols = -1; - pBuilder->nCols = -1; - pBuilder->flen = -1; - pBuilder->pBitmap = NULL; -} - -// internal func -static FORCE_INLINE int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, - bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset) { - if ((offset < (int32_t)sizeof(TSKEY)) || (colIdx < 1)) { - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - offset -= sizeof(TSKEY); - --colIdx; - -#ifdef TD_SUPPORT_BITMAP - if (tdSetBitmapValType(pBuilder->pBitmap, colIdx, valType, 0) != TSDB_CODE_SUCCESS) { - return terrno; - } -#endif - - STSRow *row = pBuilder->pBuf; - - // 1. No need to set flen part for Null/None, just use bitmap. When upsert for the same primary TS key, the bitmap - // should be updated simultaneously if Norm val overwrite Null/None cols. - // 2. When consume STSRow in memory by taos client/tq, the output of Null/None cols should both be Null. - if (tdValIsNorm(valType, val, colType)) { - // TODO: The layout of new data types imported since 3.0 like blob/medium blob is the same with binary/nchar. - if (IS_VAR_DATA_TYPE(colType)) { - // ts key stored in STSRow.ts - *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(row), offset) = TD_ROW_LEN(row); - if (isCopyVarData) { - memcpy(POINTER_SHIFT(row, TD_ROW_LEN(row)), val, varDataTLen(val)); - } - TD_ROW_LEN(row) += varDataTLen(val); - } else { - memcpy(POINTER_SHIFT(TD_ROW_DATA(row), offset), val, TYPE_BYTES[colType]); - } - } -#ifdef TD_SUPPORT_BACK2 - // NULL/None value - else { - // TODO: Null value for new data types imported since 3.0 need to be defined. - const void *nullVal = getNullValue(colType); - if (IS_VAR_DATA_TYPE(colType)) { - // ts key stored in STSRow.ts - *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(row), offset) = TD_ROW_LEN(row); - - if (isCopyVarData) { - memcpy(POINTER_SHIFT(row, TD_ROW_LEN(row)), nullVal, varDataTLen(nullVal)); - } - TD_ROW_LEN(row) += varDataTLen(nullVal); - } else { - memcpy(POINTER_SHIFT(TD_ROW_DATA(row), offset), nullVal, TYPE_BYTES[colType]); - } - } -#endif - - return 0; -} - -// internal func -static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, - bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset, - col_id_t colId) { - if ((offset < (int32_t)sizeof(SKvRowIdx)) || (colIdx < 1)) { - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - offset -= sizeof(SKvRowIdx); - --colIdx; - -#ifdef TD_SUPPORT_BITMAP - if (tdSetBitmapValType(pBuilder->pBitmap, colIdx, valType, 0) != TSDB_CODE_SUCCESS) { - return terrno; - } -#endif - - STSRow *row = pBuilder->pBuf; - // No need to store None/Null values. - if (tdValIsNorm(valType, val, colType)) { - // ts key stored in STSRow.ts - SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); - char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); - pColIdx->colId = colId; - pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN - - if (IS_VAR_DATA_TYPE(colType)) { - if (isCopyVarData) { - memcpy(ptr, val, varDataTLen(val)); - } - TD_ROW_LEN(row) += varDataTLen(val); - } else { - memcpy(ptr, val, TYPE_BYTES[colType]); - TD_ROW_LEN(row) += TYPE_BYTES[colType]; - } - } -#ifdef TD_SUPPORT_BACK2 - // NULL/None value - else { - SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); - char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); - pColIdx->colId = colId; - pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN - const void *nullVal = getNullValue(colType); - - if (IS_VAR_DATA_TYPE(colType)) { - if (isCopyVarData) { - memcpy(ptr, nullVal, varDataTLen(nullVal)); - } - TD_ROW_LEN(row) += varDataTLen(nullVal); - } else { - memcpy(ptr, nullVal, TYPE_BYTES[colType]); - TD_ROW_LEN(row) += TYPE_BYTES[colType]; - } - } -#endif - - return 0; -} - -/** - * @brief exposed func - * - * @param pBuilder - * @param colId start from PRIMARYKEY_TIMESTAMP_COL_ID - * @param colType - * @param valType - * @param val - * @param isCopyVarData - * @param offset - * @param colIdx sorted column index, start from 0 - * @return FORCE_INLINE - */ -static FORCE_INLINE int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colType, - TDRowValT valType, const void *val, bool isCopyVarData, int32_t offset, - col_id_t colIdx) { - STSRow *pRow = pBuilder->pBuf; - if (!val) { -#ifdef TD_SUPPORT_BITMAP - if (tdValTypeIsNorm(valType)) { - terrno = TSDB_CODE_INVALID_PTR; - return terrno; - } -#else - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; -#endif - } - // TS KEY is stored in STSRow.ts and not included in STSRow.data field. - if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - TD_ROW_KEY(pRow) = *(TSKEY *)val; - // The primary TS key is Norm all the time, thus its valType is not stored in bitmap. - return TSDB_CODE_SUCCESS; - } - // TODO: We can avoid the type judegement by FP, but would prevent the inline scheme. - if (TD_IS_TP_ROW(pRow)) { - tdAppendColValToTpRow(pBuilder, valType, val, isCopyVarData, colType, colIdx, offset); - } else { - tdAppendColValToKvRow(pBuilder, valType, val, isCopyVarData, colType, colIdx, offset, colId); - } - return TSDB_CODE_SUCCESS; -} - -// internal -static FORCE_INLINE int32_t tdGetTpRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int8_t colType, - int32_t offset, int16_t colIdx) { -#ifdef TD_SUPPORT_BITMAP - if (tdGetBitmapValType(pBitmap, colIdx, &output->valType, 0) != TSDB_CODE_SUCCESS) { - output->valType = TD_VTYPE_NONE; - return terrno; - } - if (tdValTypeIsNorm(output->valType)) { - if (IS_VAR_DATA_TYPE(colType)) { - output->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); - } else { - output->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); - } - } -#else - if (IS_VAR_DATA_TYPE(colType)) { - output->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); - } else { - output->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); - } - output->valType = isNull(output->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM; -#endif - return TSDB_CODE_SUCCESS; -} - -static FORCE_INLINE int32_t compareKvRowColId(const void *key1, const void *key2) { - if (*(int16_t *)key1 > ((SColIdx *)key2)->colId) { - return 1; - } else if (*(int16_t *)key1 < ((SColIdx *)key2)->colId) { - return -1; - } else { - return 0; - } -} -// internal -static FORCE_INLINE int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_t offset, - int16_t colIdx) { -#ifdef TD_SUPPORT_BITMAP - TASSERT(colIdx < tdRowGetNCols(pRow) - 1); - if (tdGetBitmapValType(pBitmap, colIdx, &output->valType, 0) != TSDB_CODE_SUCCESS) { - output->valType = TD_VTYPE_NONE; - return terrno; - } - if (tdValTypeIsNorm(output->valType)) { - if (offset < 0) { - terrno = TSDB_CODE_INVALID_PARA; - output->valType = TD_VTYPE_NONE; - return terrno; - } - output->val = POINTER_SHIFT(pRow, offset); - } -#else - TASSERT(0); - if (offset < 0) { - terrno = TSDB_CODE_INVALID_PARA; - output->valType = TD_VTYPE_NONE; - return terrno; - } - output->val = POINTER_SHIFT(pRow, offset); - output->valType = isNull(output->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM; -#endif - return TSDB_CODE_SUCCESS; -} +int32_t tdSRowSetInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen); +int32_t tdSRowSetTpInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t flen); +int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen, + int32_t allNullLen, int32_t boundNullLen); +int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf); +int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf); +int32_t tdSRowInitEx(SRowBuilder *pBuilder, void *pBuf, uint32_t allNullLen, uint32_t boundNullLen, int32_t nCols, + int32_t nBoundCols, int32_t flen); +void tdSRowReset(SRowBuilder *pBuilder); +int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, + int8_t colType, int16_t colIdx, int32_t offset); +int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, + int8_t colType, int16_t colIdx, int32_t offset, col_id_t colId); +int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colType, TDRowValT valType, const void *val, + bool isCopyVarData, int32_t offset, col_id_t colIdx); +int32_t tdGetTpRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int8_t colType, int32_t offset, + int16_t colIdx); +int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_t offset, int16_t colIdx); typedef struct { STSchema *pSchema; @@ -1026,390 +319,20 @@ typedef struct { col_id_t kvIdx; // [0, nKvCols) } STSRowIter; -static FORCE_INLINE void tdSTSRowIterReset(STSRowIter *pIter, STSRow *pRow) { - pIter->pRow = pRow; - pIter->pBitmap = tdGetBitmapAddr(pRow, pRow->type, pIter->pSchema->flen, tdRowGetNCols(pRow)); - pIter->offset = 0; - pIter->colIdx = PRIMARYKEY_TIMESTAMP_COL_ID; - pIter->kvIdx = 0; -} - -static FORCE_INLINE void tdSTSRowIterInit(STSRowIter *pIter, STSchema *pSchema) { - pIter->pSchema = pSchema; - pIter->maxColId = pSchema->columns[pSchema->numOfCols - 1].colId; -} - -static int32_t tdCompareColId(const void *arg1, const void *arg2) { - int32_t colId = *(int32_t *)arg1; - STColumn *pCol = (STColumn *)arg2; - - if (colId < pCol->colId) { - return -1; - } else if (colId == pCol->colId) { - return 0; - } else { - return 1; - } -} - -/** - * @brief STSRow method to get value of specified colId/colType by bsearch - * - * @param pIter - * @param colId Start from PRIMARYKEY_TIMESTAMP_COL_ID(1) - * @param colType - * @param pVal - * @return true Not reach end and pVal is set(None/Null/Norm). - * @return false Reach end and pVal not set. - */ -static FORCE_INLINE bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal) { - if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - pVal->val = &pIter->pRow->ts; - pVal->valType = TD_VTYPE_NORM; - return true; - } - - STSRow *pRow = pIter->pRow; - int16_t colIdx = -1; - if (TD_IS_TP_ROW(pRow)) { - STSchema *pSchema = pIter->pSchema; - STColumn *pCol = - (STColumn *)taosbsearch(&colId, pSchema->columns, pSchema->numOfCols, sizeof(STColumn), tdCompareColId, TD_EQ); - if (!pCol) { - pVal->valType = TD_VTYPE_NONE; - if (COL_REACH_END(colId, pIter->maxColId)) return false; - return true; - } -#ifdef TD_SUPPORT_BITMAP - colIdx = POINTER_DISTANCE(pCol, pSchema->columns) / sizeof(STColumn); -#endif - tdGetTpRowValOfCol(pVal, pRow, pIter->pBitmap, pCol->type, pCol->offset - sizeof(TSKEY), colIdx - 1); - } else if (TD_IS_KV_ROW(pRow)) { - SKvRowIdx *pIdx = (SKvRowIdx *)taosbsearch(&colId, TD_ROW_COL_IDX(pRow), tdRowGetNCols(pRow), sizeof(SKvRowIdx), - compareKvRowColId, TD_EQ); -#ifdef TD_SUPPORT_BITMAP - if (pIdx) { - colIdx = POINTER_DISTANCE(TD_ROW_COL_IDX(pRow), pIdx) / sizeof(SKvRowIdx); - } -#endif - tdGetKvRowValOfCol(pVal, pRow, pIter->pBitmap, pIdx ? pIdx->offset : -1, colIdx); - } else { - if (COL_REACH_END(colId, pIter->maxColId)) return false; - pVal->valType = TD_VTYPE_NONE; - } - - return true; -} - -// internal -static FORCE_INLINE bool tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colType, int32_t offset, SCellVal *pVal) { - STSRow *pRow = pIter->pRow; - if (IS_VAR_DATA_TYPE(colType)) { - pVal->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); - } else { - pVal->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); - } - -#ifdef TD_SUPPORT_BITMAP - if (tdGetBitmapValType(pIter->pBitmap, pIter->colIdx - 1, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { - pVal->valType = TD_VTYPE_NONE; - } -#else - pVal->valType = isNull(pVal->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM; -#endif - - return true; -} - -// internal -static FORCE_INLINE bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_type_t colType, col_id_t *nIdx, - SCellVal *pVal) { - STSRow *pRow = pIter->pRow; - SKvRowIdx *pKvIdx = NULL; - bool colFound = false; - col_id_t kvNCols = tdRowGetNCols(pRow) - 1; - while (*nIdx < kvNCols) { - pKvIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(pRow), *nIdx * sizeof(SKvRowIdx)); - if (pKvIdx->colId == colId) { - ++(*nIdx); - pVal->val = POINTER_SHIFT(pRow, pKvIdx->offset); - colFound = true; - break; - } else if (pKvIdx->colId > colId) { - pVal->valType = TD_VTYPE_NONE; - return true; - } else { - ++(*nIdx); - } - } - - if (!colFound) { - if (colId <= pIter->maxColId) { - pVal->valType = TD_VTYPE_NONE; - return true; - } else { - return false; - } - } - -#ifdef TD_SUPPORT_BITMAP - int16_t colIdx = -1; - if (pKvIdx) colIdx = POINTER_DISTANCE(TD_ROW_COL_IDX(pRow), pKvIdx) / sizeof(SKvRowIdx); - if (tdGetBitmapValType(pIter->pBitmap, colIdx, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { - pVal->valType = TD_VTYPE_NONE; - } -#else - pVal->valType = isNull(pVal->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM; -#endif - - return true; -} - -/** - * @brief STSRow Iter to get value from colId 1 to maxColId ascendingly - * - * @param pIter - * @param pVal - * @param colId - * @param colType - * @param pVal output - * @return true Not reach end and pVal is set(None/Null/Norm). - * @return false Reach end of row and pVal not set. - */ -static FORCE_INLINE bool tdSTSRowIterNext(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal) { - if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - pVal->val = &pIter->pRow->ts; - pVal->valType = TD_VTYPE_NORM; - return true; - } - - if (TD_IS_TP_ROW(pIter->pRow)) { - STColumn *pCol = NULL; - STSchema *pSchema = pIter->pSchema; - while (pIter->colIdx < pSchema->numOfCols) { - pCol = &pSchema->columns[pIter->colIdx]; // 1st column of schema is primary TS key - if (colId == pCol->colId) { - break; - } else if (pCol->colId < colId) { - ++pIter->colIdx; - continue; - } else { - return false; - } - } - tdGetTpRowDataOfCol(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal); - ++pIter->colIdx; - } else if (TD_IS_KV_ROW(pIter->pRow)) { - return tdGetKvRowValOfColEx(pIter, colId, colType, &pIter->kvIdx, pVal); - } else { - pVal->valType = TD_VTYPE_NONE; - terrno = TSDB_CODE_INVALID_PARA; - if (COL_REACH_END(colId, pIter->maxColId)) return false; - } - return true; -} - +void tdSTSRowIterReset(STSRowIter *pIter, STSRow *pRow); +void tdSTSRowIterInit(STSRowIter *pIter, STSchema *pSchema); +bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal); +bool tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colType, int32_t offset, SCellVal *pVal); +bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_type_t colType, col_id_t *nIdx, SCellVal *pVal); +bool tdSTSRowIterNext(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal); STSRow *mergeTwoRows(void *buffer, STSRow *row1, STSRow *row2, STSchema *pSchema1, STSchema *pSchema2); - -// Get the data pointer from a column-wised data -static FORCE_INLINE int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, int32_t row, int8_t bitmapMode) { - if (isAllRowsNone(pCol)) { - pVal->valType = TD_VTYPE_NULL; -#ifdef TD_SUPPORT_READ2 - pVal->val = (void *)getNullValue(pCol->type); -#else - pVal->val = NULL; -#endif - return TSDB_CODE_SUCCESS; - } - - if (TD_COL_ROWS_NORM(pCol)) { - pVal->valType = TD_VTYPE_NORM; - } else if (tdGetBitmapValType(pCol->pBitmap, row, &(pVal->valType), bitmapMode) < 0) { - return terrno; - } - - if (tdValTypeIsNorm(pVal->valType)) { - if (IS_VAR_DATA_TYPE(pCol->type)) { - pVal->val = POINTER_SHIFT(pCol->pData, pCol->dataOff[row]); - } else { - pVal->val = POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row); - } - } else { - pVal->valType = TD_VTYPE_NULL; -#ifdef TD_SUPPORT_READ2 - pVal->val = (void *)getNullValue(pCol->type); -#else - pVal->val = NULL; -#endif - } - return TSDB_CODE_SUCCESS; -} - -/** - * @brief - * - * @param pRow - * @param colId - * @param colType - * @param flen - * @param offset - * @param colIdx start from 0 - * @param pVal - * @return FORCE_INLINE - */ -static FORCE_INLINE bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t flen, uint32_t offset, - col_id_t colIdx, SCellVal *pVal) { - if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - tdRowSetVal(pVal, TD_VTYPE_NORM, TD_ROW_KEY_ADDR(pRow)); - return true; - } - void *pBitmap = tdGetBitmapAddrTp(pRow, flen); - tdGetTpRowValOfCol(pVal, pRow, pBitmap, colType, offset - sizeof(TSKEY), colIdx); - return true; -} - -/** - * @brief - * - * @param pRow - * @param colId - * @param offset - * @param colIdx start from 0 - * @param pVal - * @return FORCE_INLINE - */ -static FORCE_INLINE bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, uint32_t offset, col_id_t colIdx, - SCellVal *pVal) { - if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - tdRowSetVal(pVal, TD_VTYPE_NORM, TD_ROW_KEY_ADDR(pRow)); - return true; - } - void *pBitmap = tdGetBitmapAddrKv(pRow, tdRowGetNCols(pRow)); - tdGetKvRowValOfCol(pVal, pRow, pBitmap, offset, colIdx); - return true; -} - -static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int32_t rows, int8_t bitmapMode) { - ASSERT(rows > 0); - int32_t result = 0; - - if (IS_VAR_DATA_TYPE(pDataCol->type)) { - result += pDataCol->dataOff[rows - 1]; - SCellVal val = {0}; - if (tdGetColDataOfRow(&val, pDataCol, rows - 1, bitmapMode) < 0) { - TASSERT(0); - } - - // Currently, count the varDataTLen in of Null/None cols considering back compatibility test for 2.4 - result += varDataTLen(val.val); - // TODO: later on, don't save Null/None for VarDataT for 3.0 - // if (tdValTypeIsNorm(val.valType)) { - // result += varDataTLen(val.val); - // } - } else { - result += TYPE_BYTES[pDataCol->type] * rows; - } - - ASSERT(pDataCol->len == result); - - return result; -} - -static void tdSCellValPrint(SCellVal *pVal, int8_t colType) { - if (tdValTypeIsNull(pVal->valType)) { - printf("NULL "); - return; - } else if (tdValTypeIsNone(pVal->valType)) { - printf("NONE "); - return; - } - switch (colType) { - case TSDB_DATA_TYPE_BOOL: - printf("%s ", (*(int8_t *)pVal->val) == 0 ? "false" : "true"); - break; - case TSDB_DATA_TYPE_TINYINT: - printf("%" PRIi8 " ", *(int8_t *)pVal->val); - break; - case TSDB_DATA_TYPE_SMALLINT: - printf("%" PRIi16 " ", *(int16_t *)pVal->val); - break; - case TSDB_DATA_TYPE_INT: - printf("%" PRIi32 " ", *(int32_t *)pVal->val); - break; - case TSDB_DATA_TYPE_BIGINT: - printf("%" PRIi64 " ", *(int64_t *)pVal->val); - break; - case TSDB_DATA_TYPE_FLOAT: - printf("%f ", *(float *)pVal->val); - break; - case TSDB_DATA_TYPE_DOUBLE: - printf("%lf ", *(double *)pVal->val); - break; - case TSDB_DATA_TYPE_VARCHAR: - printf("VARCHAR "); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - printf("%" PRIi64 " ", *(int64_t *)pVal->val); - break; - case TSDB_DATA_TYPE_NCHAR: - printf("NCHAR "); - break; - case TSDB_DATA_TYPE_UTINYINT: - printf("%" PRIu8 " ", *(uint8_t *)pVal->val); - break; - case TSDB_DATA_TYPE_USMALLINT: - printf("%" PRIu16 " ", *(uint16_t *)pVal->val); - break; - case TSDB_DATA_TYPE_UINT: - printf("%" PRIu32 " ", *(uint32_t *)pVal->val); - break; - case TSDB_DATA_TYPE_UBIGINT: - printf("%" PRIu64 " ", *(uint64_t *)pVal->val); - break; - case TSDB_DATA_TYPE_JSON: - printf("JSON "); - break; - case TSDB_DATA_TYPE_VARBINARY: - printf("VARBIN "); - break; - case TSDB_DATA_TYPE_DECIMAL: - printf("DECIMAL "); - break; - case TSDB_DATA_TYPE_BLOB: - printf("BLOB "); - break; - case TSDB_DATA_TYPE_MEDIUMBLOB: - printf("MedBLOB "); - break; - // case TSDB_DATA_TYPE_BINARY: - // printf("BINARY "); - // break; - case TSDB_DATA_TYPE_MAX: - printf("UNDEF "); - break; - default: - printf("UNDEF "); - break; - } -} - -static void tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag) { - STSRowIter iter = {0}; - tdSTSRowIterInit(&iter, pSchema); - tdSTSRowIterReset(&iter, row); - printf("%s >>>", tag); - for (int i = 0; i < pSchema->numOfCols; ++i) { - STColumn *stCol = pSchema->columns + i; - SCellVal sVal = {255, NULL}; - if (!tdSTSRowIterNext(&iter, stCol->colId, stCol->type, &sVal)) { - break; - } - ASSERT(sVal.valType == 0 || sVal.valType == 1 || sVal.valType == 2); - tdSCellValPrint(&sVal, stCol->type); - } - printf("\n"); -} +int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, int32_t row, int8_t bitmapMode); +bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t flen, uint32_t offset, col_id_t colIdx, + SCellVal *pVal); +bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, uint32_t offset, col_id_t colIdx, SCellVal *pVal); +int32_t dataColGetNEleLen(SDataCol *pDataCol, int32_t rows, int8_t bitmapMode); +void tdSCellValPrint(SCellVal *pVal, int8_t colType); +void tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag); #ifdef __cplusplus } diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 50e51a40dd..44bcd72a33 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -33,6 +33,7 @@ const uint8_t tdVTypeByte[2][3] = {{ // declaration static uint8_t tdGetBitmapByte(uint8_t byte); +static int32_t tdCompareColId(const void *arg1, const void *arg2); // static void dataColSetNEleNull(SDataCol *pCol, int nEle); @@ -916,4 +917,937 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { } return pRet; +} + +void tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag) { + STSRowIter iter = {0}; + tdSTSRowIterInit(&iter, pSchema); + tdSTSRowIterReset(&iter, row); + printf("%s >>>", tag); + for (int i = 0; i < pSchema->numOfCols; ++i) { + STColumn *stCol = pSchema->columns + i; + SCellVal sVal = {255, NULL}; + if (!tdSTSRowIterNext(&iter, stCol->colId, stCol->type, &sVal)) { + break; + } + ASSERT(sVal.valType == 0 || sVal.valType == 1 || sVal.valType == 2); + tdSCellValPrint(&sVal, stCol->type); + } + printf("\n"); +} + +void tdSCellValPrint(SCellVal *pVal, int8_t colType) { + if (tdValTypeIsNull(pVal->valType)) { + printf("NULL "); + return; + } else if (tdValTypeIsNone(pVal->valType)) { + printf("NONE "); + return; + } + switch (colType) { + case TSDB_DATA_TYPE_BOOL: + printf("%s ", (*(int8_t *)pVal->val) == 0 ? "false" : "true"); + break; + case TSDB_DATA_TYPE_TINYINT: + printf("%" PRIi8 " ", *(int8_t *)pVal->val); + break; + case TSDB_DATA_TYPE_SMALLINT: + printf("%" PRIi16 " ", *(int16_t *)pVal->val); + break; + case TSDB_DATA_TYPE_INT: + printf("%" PRIi32 " ", *(int32_t *)pVal->val); + break; + case TSDB_DATA_TYPE_BIGINT: + printf("%" PRIi64 " ", *(int64_t *)pVal->val); + break; + case TSDB_DATA_TYPE_FLOAT: + printf("%f ", *(float *)pVal->val); + break; + case TSDB_DATA_TYPE_DOUBLE: + printf("%lf ", *(double *)pVal->val); + break; + case TSDB_DATA_TYPE_VARCHAR: + printf("VARCHAR "); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + printf("%" PRIi64 " ", *(int64_t *)pVal->val); + break; + case TSDB_DATA_TYPE_NCHAR: + printf("NCHAR "); + break; + case TSDB_DATA_TYPE_UTINYINT: + printf("%" PRIu8 " ", *(uint8_t *)pVal->val); + break; + case TSDB_DATA_TYPE_USMALLINT: + printf("%" PRIu16 " ", *(uint16_t *)pVal->val); + break; + case TSDB_DATA_TYPE_UINT: + printf("%" PRIu32 " ", *(uint32_t *)pVal->val); + break; + case TSDB_DATA_TYPE_UBIGINT: + printf("%" PRIu64 " ", *(uint64_t *)pVal->val); + break; + case TSDB_DATA_TYPE_JSON: + printf("JSON "); + break; + case TSDB_DATA_TYPE_VARBINARY: + printf("VARBIN "); + break; + case TSDB_DATA_TYPE_DECIMAL: + printf("DECIMAL "); + break; + case TSDB_DATA_TYPE_BLOB: + printf("BLOB "); + break; + case TSDB_DATA_TYPE_MEDIUMBLOB: + printf("MedBLOB "); + break; + // case TSDB_DATA_TYPE_BINARY: + // printf("BINARY "); + // break; + case TSDB_DATA_TYPE_MAX: + printf("UNDEF "); + break; + default: + printf("UNDEF "); + break; + } +} + +int32_t dataColGetNEleLen(SDataCol *pDataCol, int32_t rows, int8_t bitmapMode) { + ASSERT(rows > 0); + int32_t result = 0; + + if (IS_VAR_DATA_TYPE(pDataCol->type)) { + result += pDataCol->dataOff[rows - 1]; + SCellVal val = {0}; + if (tdGetColDataOfRow(&val, pDataCol, rows - 1, bitmapMode) < 0) { + TASSERT(0); + } + + // Currently, count the varDataTLen in of Null/None cols considering back compatibility test for 2.4 + result += varDataTLen(val.val); + // TODO: later on, don't save Null/None for VarDataT for 3.0 + // if (tdValTypeIsNorm(val.valType)) { + // result += varDataTLen(val.val); + // } + } else { + result += TYPE_BYTES[pDataCol->type] * rows; + } + + ASSERT(pDataCol->len == result); + + return result; +} + +bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, uint32_t offset, col_id_t colIdx, SCellVal *pVal) { + if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + tdRowSetVal(pVal, TD_VTYPE_NORM, TD_ROW_KEY_ADDR(pRow)); + return true; + } + void *pBitmap = tdGetBitmapAddrKv(pRow, tdRowGetNCols(pRow)); + tdGetKvRowValOfCol(pVal, pRow, pBitmap, offset, colIdx); + return true; +} + +bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t flen, uint32_t offset, col_id_t colIdx, + SCellVal *pVal) { + if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + tdRowSetVal(pVal, TD_VTYPE_NORM, TD_ROW_KEY_ADDR(pRow)); + return true; + } + void *pBitmap = tdGetBitmapAddrTp(pRow, flen); + tdGetTpRowValOfCol(pVal, pRow, pBitmap, colType, offset - sizeof(TSKEY), colIdx); + return true; +} + +int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, int32_t row, int8_t bitmapMode) { + if (isAllRowsNone(pCol)) { + pVal->valType = TD_VTYPE_NULL; +#ifdef TD_SUPPORT_READ2 + pVal->val = (void *)getNullValue(pCol->type); +#else + pVal->val = NULL; +#endif + return TSDB_CODE_SUCCESS; + } + + if (TD_COL_ROWS_NORM(pCol)) { + pVal->valType = TD_VTYPE_NORM; + } else if (tdGetBitmapValType(pCol->pBitmap, row, &(pVal->valType), bitmapMode) < 0) { + return terrno; + } + + if (tdValTypeIsNorm(pVal->valType)) { + if (IS_VAR_DATA_TYPE(pCol->type)) { + pVal->val = POINTER_SHIFT(pCol->pData, pCol->dataOff[row]); + } else { + pVal->val = POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row); + } + } else { + pVal->valType = TD_VTYPE_NULL; +#ifdef TD_SUPPORT_READ2 + pVal->val = (void *)getNullValue(pCol->type); +#else + pVal->val = NULL; +#endif + } + return TSDB_CODE_SUCCESS; +} + +bool tdSTSRowIterNext(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal) { + if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + pVal->val = &pIter->pRow->ts; + pVal->valType = TD_VTYPE_NORM; + return true; + } + + if (TD_IS_TP_ROW(pIter->pRow)) { + STColumn *pCol = NULL; + STSchema *pSchema = pIter->pSchema; + while (pIter->colIdx < pSchema->numOfCols) { + pCol = &pSchema->columns[pIter->colIdx]; // 1st column of schema is primary TS key + if (colId == pCol->colId) { + break; + } else if (pCol->colId < colId) { + ++pIter->colIdx; + continue; + } else { + return false; + } + } + tdGetTpRowDataOfCol(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal); + ++pIter->colIdx; + } else if (TD_IS_KV_ROW(pIter->pRow)) { + return tdGetKvRowValOfColEx(pIter, colId, colType, &pIter->kvIdx, pVal); + } else { + pVal->valType = TD_VTYPE_NONE; + terrno = TSDB_CODE_INVALID_PARA; + if (COL_REACH_END(colId, pIter->maxColId)) return false; + } + return true; +} + +bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_type_t colType, col_id_t *nIdx, SCellVal *pVal) { + STSRow *pRow = pIter->pRow; + SKvRowIdx *pKvIdx = NULL; + bool colFound = false; + col_id_t kvNCols = tdRowGetNCols(pRow) - 1; + while (*nIdx < kvNCols) { + pKvIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(pRow), *nIdx * sizeof(SKvRowIdx)); + if (pKvIdx->colId == colId) { + ++(*nIdx); + pVal->val = POINTER_SHIFT(pRow, pKvIdx->offset); + colFound = true; + break; + } else if (pKvIdx->colId > colId) { + pVal->valType = TD_VTYPE_NONE; + return true; + } else { + ++(*nIdx); + } + } + + if (!colFound) { + if (colId <= pIter->maxColId) { + pVal->valType = TD_VTYPE_NONE; + return true; + } else { + return false; + } + } + +#ifdef TD_SUPPORT_BITMAP + int16_t colIdx = -1; + if (pKvIdx) colIdx = POINTER_DISTANCE(TD_ROW_COL_IDX(pRow), pKvIdx) / sizeof(SKvRowIdx); + if (tdGetBitmapValType(pIter->pBitmap, colIdx, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { + pVal->valType = TD_VTYPE_NONE; + } +#else + pVal->valType = isNull(pVal->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM; +#endif + + return true; +} + +bool tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colType, int32_t offset, SCellVal *pVal) { + STSRow *pRow = pIter->pRow; + if (IS_VAR_DATA_TYPE(colType)) { + pVal->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); + } else { + pVal->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); + } + +#ifdef TD_SUPPORT_BITMAP + if (tdGetBitmapValType(pIter->pBitmap, pIter->colIdx - 1, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { + pVal->valType = TD_VTYPE_NONE; + } +#else + pVal->valType = isNull(pVal->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM; +#endif + + return true; +} + +static FORCE_INLINE int32_t compareKvRowColId(const void *key1, const void *key2) { + if (*(int16_t *)key1 > ((SColIdx *)key2)->colId) { + return 1; + } else if (*(int16_t *)key1 < ((SColIdx *)key2)->colId) { + return -1; + } else { + return 0; + } +} + +bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal) { + if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + pVal->val = &pIter->pRow->ts; + pVal->valType = TD_VTYPE_NORM; + return true; + } + + STSRow *pRow = pIter->pRow; + int16_t colIdx = -1; + if (TD_IS_TP_ROW(pRow)) { + STSchema *pSchema = pIter->pSchema; + STColumn *pCol = + (STColumn *)taosbsearch(&colId, pSchema->columns, pSchema->numOfCols, sizeof(STColumn), tdCompareColId, TD_EQ); + if (!pCol) { + pVal->valType = TD_VTYPE_NONE; + if (COL_REACH_END(colId, pIter->maxColId)) return false; + return true; + } +#ifdef TD_SUPPORT_BITMAP + colIdx = POINTER_DISTANCE(pCol, pSchema->columns) / sizeof(STColumn); +#endif + tdGetTpRowValOfCol(pVal, pRow, pIter->pBitmap, pCol->type, pCol->offset - sizeof(TSKEY), colIdx - 1); + } else if (TD_IS_KV_ROW(pRow)) { + SKvRowIdx *pIdx = (SKvRowIdx *)taosbsearch(&colId, TD_ROW_COL_IDX(pRow), tdRowGetNCols(pRow), sizeof(SKvRowIdx), + compareKvRowColId, TD_EQ); +#ifdef TD_SUPPORT_BITMAP + if (pIdx) { + colIdx = POINTER_DISTANCE(TD_ROW_COL_IDX(pRow), pIdx) / sizeof(SKvRowIdx); + } +#endif + tdGetKvRowValOfCol(pVal, pRow, pIter->pBitmap, pIdx ? pIdx->offset : -1, colIdx); + } else { + if (COL_REACH_END(colId, pIter->maxColId)) return false; + pVal->valType = TD_VTYPE_NONE; + } + + return true; +} + +static int32_t tdCompareColId(const void *arg1, const void *arg2) { + int32_t colId = *(int32_t *)arg1; + STColumn *pCol = (STColumn *)arg2; + + if (colId < pCol->colId) { + return -1; + } else if (colId == pCol->colId) { + return 0; + } else { + return 1; + } +} + +int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) { + if (!pBitmap || colIdx < 0) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + int16_t nBytes = colIdx / TD_VTYPE_PARTS; + int16_t nOffset = colIdx & TD_VTYPE_OPTR; + char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); + // use literal value directly and not use formula to simplify the codes + switch (nOffset) { + case 0: + *pValType = (((*pDestByte) & 0xC0) >> 6); + break; + case 1: + *pValType = (((*pDestByte) & 0x30) >> 4); + break; + case 2: + *pValType = (((*pDestByte) & 0x0C) >> 2); + break; + case 3: + *pValType = ((*pDestByte) & 0x03); + break; + default: + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + return TSDB_CODE_SUCCESS; +} + +int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) { + if (!pBitmap || colIdx < 0) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + int16_t nBytes = colIdx / TD_VTYPE_PARTS_I; + int16_t nOffset = colIdx & TD_VTYPE_OPTR_I; + char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); + // use literal value directly and not use formula to simplify the codes + switch (nOffset) { + case 0: + *pValType = (((*pDestByte) & 0x80) >> 7); + break; + case 1: + *pValType = (((*pDestByte) & 0x40) >> 6); + break; + case 2: + *pValType = (((*pDestByte) & 0x20) >> 5); + break; + case 3: + *pValType = (((*pDestByte) & 0x10) >> 4); + break; + case 4: + *pValType = (((*pDestByte) & 0x08) >> 3); + break; + case 5: + *pValType = (((*pDestByte) & 0x04) >> 2); + break; + case 6: + *pValType = (((*pDestByte) & 0x02) >> 1); + break; + case 7: + *pValType = ((*pDestByte) & 0x01); + break; + default: + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + return TSDB_CODE_SUCCESS; +} + +int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType) { + if (!pBitmap || colIdx < 0) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + int16_t nBytes = colIdx / TD_VTYPE_PARTS_I; + int16_t nOffset = colIdx & TD_VTYPE_OPTR_I; + char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); + // use literal value directly and not use formula to simplify the codes + switch (nOffset) { + case 0: + *pDestByte = ((*pDestByte) & 0x7F) | (valType << 7); + // set the value and clear other partitions for offset 0 + // *pDestByte |= (valType << 7); + break; + case 1: + *pDestByte = ((*pDestByte) & 0xBF) | (valType << 6); + // *pDestByte |= (valType << 6); + break; + case 2: + *pDestByte = ((*pDestByte) & 0xDF) | (valType << 5); + // *pDestByte |= (valType << 5); + break; + case 3: + *pDestByte = ((*pDestByte) & 0xEF) | (valType << 4); + // *pDestByte |= (valType << 4); + break; + case 4: + *pDestByte = ((*pDestByte) & 0xF7) | (valType << 3); + // *pDestByte |= (valType << 3); + break; + case 5: + *pDestByte = ((*pDestByte) & 0xFB) | (valType << 2); + // *pDestByte |= (valType << 2); + break; + case 6: + *pDestByte = ((*pDestByte) & 0xFD) | (valType << 1); + // *pDestByte |= (valType << 1); + break; + case 7: + *pDestByte = ((*pDestByte) & 0xFE) | valType; + // *pDestByte |= (valType); + break; + default: + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + return TSDB_CODE_SUCCESS; +} + +int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_t offset, int16_t colIdx) { +#ifdef TD_SUPPORT_BITMAP + TASSERT(colIdx < tdRowGetNCols(pRow) - 1); + if (tdGetBitmapValType(pBitmap, colIdx, &output->valType, 0) != TSDB_CODE_SUCCESS) { + output->valType = TD_VTYPE_NONE; + return terrno; + } + if (tdValTypeIsNorm(output->valType)) { + if (offset < 0) { + terrno = TSDB_CODE_INVALID_PARA; + output->valType = TD_VTYPE_NONE; + return terrno; + } + output->val = POINTER_SHIFT(pRow, offset); + } +#else + TASSERT(0); + if (offset < 0) { + terrno = TSDB_CODE_INVALID_PARA; + output->valType = TD_VTYPE_NONE; + return terrno; + } + output->val = POINTER_SHIFT(pRow, offset); + output->valType = isNull(output->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM; +#endif + return TSDB_CODE_SUCCESS; +} + +int32_t tdGetTpRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int8_t colType, int32_t offset, + int16_t colIdx) { +#ifdef TD_SUPPORT_BITMAP + if (tdGetBitmapValType(pBitmap, colIdx, &output->valType, 0) != TSDB_CODE_SUCCESS) { + output->valType = TD_VTYPE_NONE; + return terrno; + } + if (tdValTypeIsNorm(output->valType)) { + if (IS_VAR_DATA_TYPE(colType)) { + output->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); + } else { + output->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); + } + } +#else + if (IS_VAR_DATA_TYPE(colType)) { + output->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); + } else { + output->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); + } + output->valType = isNull(output->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM; +#endif + return TSDB_CODE_SUCCESS; +} + +int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colType, TDRowValT valType, const void *val, + bool isCopyVarData, int32_t offset, col_id_t colIdx) { + STSRow *pRow = pBuilder->pBuf; + if (!val) { +#ifdef TD_SUPPORT_BITMAP + if (tdValTypeIsNorm(valType)) { + terrno = TSDB_CODE_INVALID_PTR; + return terrno; + } +#else + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; +#endif + } + // TS KEY is stored in STSRow.ts and not included in STSRow.data field. + if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + TD_ROW_KEY(pRow) = *(TSKEY *)val; + // The primary TS key is Norm all the time, thus its valType is not stored in bitmap. + return TSDB_CODE_SUCCESS; + } + // TODO: We can avoid the type judegement by FP, but would prevent the inline scheme. + if (TD_IS_TP_ROW(pRow)) { + tdAppendColValToTpRow(pBuilder, valType, val, isCopyVarData, colType, colIdx, offset); + } else { + tdAppendColValToKvRow(pBuilder, valType, val, isCopyVarData, colType, colIdx, offset, colId); + } + return TSDB_CODE_SUCCESS; +} + +int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, + int8_t colType, int16_t colIdx, int32_t offset, col_id_t colId) { + if ((offset < (int32_t)sizeof(SKvRowIdx)) || (colIdx < 1)) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + offset -= sizeof(SKvRowIdx); + --colIdx; + +#ifdef TD_SUPPORT_BITMAP + if (tdSetBitmapValType(pBuilder->pBitmap, colIdx, valType, 0) != TSDB_CODE_SUCCESS) { + return terrno; + } +#endif + + STSRow *row = pBuilder->pBuf; + // No need to store None/Null values. + if (tdValIsNorm(valType, val, colType)) { + // ts key stored in STSRow.ts + SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); + char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); + pColIdx->colId = colId; + pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN + + if (IS_VAR_DATA_TYPE(colType)) { + if (isCopyVarData) { + memcpy(ptr, val, varDataTLen(val)); + } + TD_ROW_LEN(row) += varDataTLen(val); + } else { + memcpy(ptr, val, TYPE_BYTES[colType]); + TD_ROW_LEN(row) += TYPE_BYTES[colType]; + } + } +#ifdef TD_SUPPORT_BACK2 + // NULL/None value + else { + SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); + char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); + pColIdx->colId = colId; + pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN + const void *nullVal = getNullValue(colType); + + if (IS_VAR_DATA_TYPE(colType)) { + if (isCopyVarData) { + memcpy(ptr, nullVal, varDataTLen(nullVal)); + } + TD_ROW_LEN(row) += varDataTLen(nullVal); + } else { + memcpy(ptr, nullVal, TYPE_BYTES[colType]); + TD_ROW_LEN(row) += TYPE_BYTES[colType]; + } + } +#endif + + return 0; +} + +int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, + int8_t colType, int16_t colIdx, int32_t offset) { + if ((offset < (int32_t)sizeof(TSKEY)) || (colIdx < 1)) { + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + offset -= sizeof(TSKEY); + --colIdx; + +#ifdef TD_SUPPORT_BITMAP + if (tdSetBitmapValType(pBuilder->pBitmap, colIdx, valType, 0) != TSDB_CODE_SUCCESS) { + return terrno; + } +#endif + + STSRow *row = pBuilder->pBuf; + + // 1. No need to set flen part for Null/None, just use bitmap. When upsert for the same primary TS key, the bitmap + // should be updated simultaneously if Norm val overwrite Null/None cols. + // 2. When consume STSRow in memory by taos client/tq, the output of Null/None cols should both be Null. + if (tdValIsNorm(valType, val, colType)) { + // TODO: The layout of new data types imported since 3.0 like blob/medium blob is the same with binary/nchar. + if (IS_VAR_DATA_TYPE(colType)) { + // ts key stored in STSRow.ts + *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(row), offset) = TD_ROW_LEN(row); + if (isCopyVarData) { + memcpy(POINTER_SHIFT(row, TD_ROW_LEN(row)), val, varDataTLen(val)); + } + TD_ROW_LEN(row) += varDataTLen(val); + } else { + memcpy(POINTER_SHIFT(TD_ROW_DATA(row), offset), val, TYPE_BYTES[colType]); + } + } +#ifdef TD_SUPPORT_BACK2 + // NULL/None value + else { + // TODO: Null value for new data types imported since 3.0 need to be defined. + const void *nullVal = getNullValue(colType); + if (IS_VAR_DATA_TYPE(colType)) { + // ts key stored in STSRow.ts + *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(row), offset) = TD_ROW_LEN(row); + + if (isCopyVarData) { + memcpy(POINTER_SHIFT(row, TD_ROW_LEN(row)), nullVal, varDataTLen(nullVal)); + } + TD_ROW_LEN(row) += varDataTLen(nullVal); + } else { + memcpy(POINTER_SHIFT(TD_ROW_DATA(row), offset), nullVal, TYPE_BYTES[colType]); + } + } +#endif + + return 0; +} + +int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen, + int32_t allNullLen, int32_t boundNullLen) { + if ((boundNullLen > 0) && (allNullLen > 0) && (nBoundCols > 0)) { + uint32_t tpLen = allNullLen; + uint32_t kvLen = sizeof(col_id_t) + sizeof(SKvRowIdx) * nBoundCols + boundNullLen; + if (isSelectKVRow(kvLen, tpLen)) { + pBuilder->rowType = TD_ROW_KV; + } else { + pBuilder->rowType = TD_ROW_TP; + } + + } else { + pBuilder->rowType = TD_ROW_TP; + } + + pBuilder->flen = flen; + pBuilder->nCols = nCols; + pBuilder->nBoundCols = nBoundCols; + if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } +#ifdef TD_SUPPORT_BITMAP + // the primary TS key is stored separatedly + pBuilder->nBitmaps = (col_id_t)TD_BITMAP_BYTES(pBuilder->nCols - 1); + if (nBoundCols > 0) { + pBuilder->nBoundBitmaps = (col_id_t)TD_BITMAP_BYTES(pBuilder->nBoundCols - 1); + } else { + pBuilder->nBoundBitmaps = 0; + } +#else + pBuilder->nBitmaps = 0; + pBuilder->nBoundBitmaps = 0; +#endif + return TSDB_CODE_SUCCESS; +} + +int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { + pBuilder->pBuf = (STSRow *)pBuf; + if (!pBuilder->pBuf) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + + TD_ROW_SET_INFO(pBuilder->pBuf, 0); + TD_ROW_SET_TYPE(pBuilder->pBuf, pBuilder->rowType); + + TASSERT(pBuilder->nBitmaps > 0 && pBuilder->flen > 0); + + uint32_t len = 0; + switch (pBuilder->rowType) { + case TD_ROW_TP: +#ifdef TD_SUPPORT_BITMAP + pBuilder->pBitmap = tdGetBitmapAddrTp(pBuilder->pBuf, pBuilder->flen); + memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBitmaps); +#endif + // the primary TS key is stored separatedly + len = TD_ROW_HEAD_LEN + pBuilder->flen - sizeof(TSKEY) + pBuilder->nBitmaps; + TD_ROW_SET_LEN(pBuilder->pBuf, len); + TD_ROW_SET_SVER(pBuilder->pBuf, pBuilder->sver); + break; + case TD_ROW_KV: +#ifdef TD_SUPPORT_BITMAP + pBuilder->pBitmap = tdGetBitmapAddrKv(pBuilder->pBuf, pBuilder->nBoundCols); + memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBoundBitmaps); +#endif + len = TD_ROW_HEAD_LEN + TD_ROW_NCOLS_LEN + (pBuilder->nBoundCols - 1) * sizeof(SKvRowIdx) + + pBuilder->nBoundBitmaps; // add + TD_ROW_SET_LEN(pBuilder->pBuf, len); + TD_ROW_SET_SVER(pBuilder->pBuf, pBuilder->sver); + TD_ROW_SET_NCOLS(pBuilder->pBuf, pBuilder->nBoundCols); + break; + default: + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf) { + pBuilder->pBuf = (STSRow *)pBuf; + if (!pBuilder->pBuf) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + + TASSERT(pBuilder->nBitmaps > 0 && pBuilder->flen > 0); + + uint32_t len = 0; + switch (pBuilder->rowType) { + case TD_ROW_TP: +#ifdef TD_SUPPORT_BITMAP + pBuilder->pBitmap = tdGetBitmapAddrTp(pBuilder->pBuf, pBuilder->flen); +#endif + break; + case TD_ROW_KV: +#ifdef TD_SUPPORT_BITMAP + pBuilder->pBitmap = tdGetBitmapAddrKv(pBuilder->pBuf, pBuilder->nBoundCols); +#endif + break; + default: + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + return TSDB_CODE_SUCCESS; +} + +int32_t tdSRowInitEx(SRowBuilder *pBuilder, void *pBuf, uint32_t allNullLen, uint32_t boundNullLen, int32_t nCols, + int32_t nBoundCols, int32_t flen) { + if (tdSRowSetExtendedInfo(pBuilder, allNullLen, boundNullLen, nCols, nBoundCols, flen) < 0) { + return terrno; + } + return tdSRowResetBuf(pBuilder, pBuf); +} + +void tdSRowReset(SRowBuilder *pBuilder) { + pBuilder->rowType = TD_ROW_TP; + pBuilder->pBuf = NULL; + pBuilder->nBoundCols = -1; + pBuilder->nCols = -1; + pBuilder->flen = -1; + pBuilder->pBitmap = NULL; +} + +int32_t tdSRowSetTpInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t flen) { + pBuilder->flen = flen; + pBuilder->nCols = nCols; + if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } +#ifdef TD_SUPPORT_BITMAP + // the primary TS key is stored separatedly + pBuilder->nBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nCols - 1); +#else + pBuilder->nBitmaps = 0; + pBuilder->nBoundBitmaps = 0; +#endif + return TSDB_CODE_SUCCESS; +} + +int32_t tdSRowSetInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen) { + pBuilder->flen = flen; + pBuilder->nCols = nCols; + pBuilder->nBoundCols = nBoundCols; + if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } +#ifdef TD_SUPPORT_BITMAP + // the primary TS key is stored separatedly + pBuilder->nBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nCols - 1); + if (nBoundCols > 0) { + pBuilder->nBoundBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nBoundCols - 1); + } else { + pBuilder->nBoundBitmaps = 0; + } +#else + pBuilder->nBitmaps = 0; + pBuilder->nBoundBitmaps = 0; +#endif + return TSDB_CODE_SUCCESS; +} + +int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode) { + switch (bitmapMode) { + case 0: + tdGetBitmapValTypeII(pBitmap, colIdx, pValType); + break; + case -1: + case 1: + tdGetBitmapValTypeI(pBitmap, colIdx, pValType); + break; + default: + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; +} + +bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx, int8_t bitmapMode) { + TDRowValT valType = 0; + tdGetBitmapValType(pBitmap, idx, &valType, bitmapMode); + if (tdValTypeIsNorm(valType)) { + return true; + } + return false; +} + +int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT valType) { + if (!pBitmap || colIdx < 0) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + int16_t nBytes = colIdx / TD_VTYPE_PARTS; + int16_t nOffset = colIdx & TD_VTYPE_OPTR; + char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); + // use literal value directly and not use formula to simplify the codes + switch (nOffset) { + case 0: + *pDestByte = ((*pDestByte) & 0x3F) | (valType << 6); + // set the value and clear other partitions for offset 0 + // *pDestByte |= (valType << 6); + break; + case 1: + *pDestByte = ((*pDestByte) & 0xCF) | (valType << 4); + // *pDestByte |= (valType << 4); + break; + case 2: + *pDestByte = ((*pDestByte) & 0xF3) | (valType << 2); + // *pDestByte |= (valType << 2); + break; + case 3: + *pDestByte = ((*pDestByte) & 0xFC) | valType; + // *pDestByte |= (valType); + break; + default: + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + return TSDB_CODE_SUCCESS; +} + +int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int8_t bitmapMode) { + switch (bitmapMode) { + case 0: + tdSetBitmapValTypeII(pBitmap, colIdx, valType); + break; + case -1: + case 1: + tdSetBitmapValTypeI(pBitmap, colIdx, valType); + break; + default: + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; +} + +void *tdGetBitmapAddr(STSRow *pRow, uint8_t rowType, uint32_t flen, col_id_t nKvCols) { +#ifdef TD_SUPPORT_BITMAP + switch (rowType) { + case TD_ROW_TP: + return tdGetBitmapAddrTp(pRow, flen); + case TD_ROW_KV: + return tdGetBitmapAddrKv(pRow, nKvCols); + default: + break; + } +#endif + return NULL; +} + +void tdSTSRowIterReset(STSRowIter *pIter, STSRow *pRow) { + pIter->pRow = pRow; + pIter->pBitmap = tdGetBitmapAddr(pRow, pRow->type, pIter->pSchema->flen, tdRowGetNCols(pRow)); + pIter->offset = 0; + pIter->colIdx = PRIMARYKEY_TIMESTAMP_COL_ID; + pIter->kvIdx = 0; +} + +void tdSTSRowIterInit(STSRowIter *pIter, STSchema *pSchema) { + pIter->pSchema = pSchema; + pIter->maxColId = pSchema->columns[pSchema->numOfCols - 1].colId; } \ No newline at end of file From 0b86cc594fa4235559e08e7aa9f3f2c206e92612 Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Mon, 9 May 2022 21:46:40 +0800 Subject: [PATCH 19/29] test: add cases to ci --- tests/system-test/fulltest.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index f713f707cb..34c3484df4 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -48,4 +48,5 @@ python3 ./test.py -f 2-query/cos.py python3 ./test.py -f 2-query/tan.py python3 ./test.py -f 2-query/arcsin.py python3 ./test.py -f 2-query/arccos.py -python3 ./test.py -f 2-query/arctan.py \ No newline at end of file +python3 ./test.py -f 2-query/arctan.py +python3 ./test.py -f 2-query/query_cols_tags_and_or.py From 5122a6e2689ac5d286f38d26d0297083db635b9f Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 9 May 2022 21:52:39 +0800 Subject: [PATCH 20/29] enh: add submit row debug log in taosd --- include/common/taosdef.h | 2 +- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 5 +++ source/dnode/vnode/src/vnd/vnodeSvr.c | 46 ++++++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/include/common/taosdef.h b/include/common/taosdef.h index e1f8832edf..5384082da3 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -89,7 +89,7 @@ extern char *qtypeStr[]; #define TSDB_PORT_HTTP 11 -#undef TD_DEBUG_PRINT_ROW +#define TD_DEBUG_PRINT_ROW #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index eaa0893f29..d514512881 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -326,6 +326,11 @@ int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlo tSkipListPutBatchByIter(pTbData->pData, &blkIter, (iter_next_fn_t)tGetSubmitBlkNext); +#ifdef TD_DEBUG_PRINT_ROW + printf("!!! %s:%d table %" PRIi64 " has %d rows in skiplist\n\n", __func__, __LINE__, pTbData->uid, + SL_SIZE(pTbData->pData)); +#endif + // Set statistics keyMax = TD_ROW_KEY(blkIter.row); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index abc112038b..a54c30fc61 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -497,6 +497,47 @@ _exit: return 0; } +static int vnodeDebugPrintSubmitMsg(SVnode *pVnode, SSubmitReq *pMsg, const char* tags) { + ASSERT(pMsg != NULL); + SSubmitMsgIter msgIter = {0}; + SMeta *pMeta = pVnode->pMeta; + SSubmitBlk *pBlock = NULL; + SSubmitBlkIter blkIter = {0}; + STSRow *row = NULL; + STSchema *pSchema = NULL; + tb_uid_t suid = 0; + + if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1; + while (true) { + if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; + if (pBlock == NULL) break; + tInitSubmitBlkIter(&msgIter, pBlock, &blkIter); + if (blkIter.row == NULL) continue; + if (!pSchema || (suid != msgIter.suid)) { + if (pSchema) { + taosMemoryFreeClear(pSchema); + } + pSchema = metaGetTbTSchema(pMeta, msgIter.suid, 0); // TODO: use the real schema + if(pSchema) { + suid = msgIter.suid; + } + } + if(!pSchema) { + printf("%s:%d no valid schema\n", tags, __LINE__); + continue; + } + char __tags[128] = {0}; + snprintf(__tags, 128, "%s: uid %" PRIi64 " ", tags, msgIter.uid); + while ((row = tGetSubmitBlkNext(&blkIter))) { + tdSRowPrint(row, pSchema, __tags); + } + } + + taosMemoryFreeClear(pSchema); + + return 0; +} + static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; SSubmitMsgIter msgIter = {0}; @@ -508,6 +549,10 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in pRsp->code = 0; +#ifdef TD_DEBUG_PRINT_ROW + vnodeDebugPrintSubmitMsg(pVnode, pReq, __func__); +#endif + // handle the request if (tInitSubmitMsgIter(pSubmitReq, &msgIter) < 0) { pRsp->code = TSDB_CODE_INVALID_MSG; @@ -551,6 +596,7 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in } rsp.affectedRows += nRows; + } _exit: From 5588aadc27e0fcb8a7a507f0893414638542a382 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Mon, 9 May 2022 23:10:25 +0800 Subject: [PATCH 21/29] modify testcase that using taosBenchmark to test multi-process table building --- .../1-insert/insertWithMoreVgroup.py | 22 ++++++++++++++----- tests/system-test/1-insert/manyVgroups.json | 4 ++-- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/tests/system-test/1-insert/insertWithMoreVgroup.py b/tests/system-test/1-insert/insertWithMoreVgroup.py index ae450ae69a..d8050c53c5 100644 --- a/tests/system-test/1-insert/insertWithMoreVgroup.py +++ b/tests/system-test/1-insert/insertWithMoreVgroup.py @@ -199,7 +199,8 @@ class TDTestCase: os.system("%s -f %s -y " %(taosBenchbin,jsonFile)) return - def taosBenchCreate(self,dbname,stbname,vgroups,threadNumbers,count): + def taosBenchCreate(self,host,dropdb,dbname,stbname,vgroups,threadNumbers,count): + # count=50000 buildPath = self.getBuildPath() if (buildPath == ""): @@ -207,12 +208,18 @@ class TDTestCase: else: tdLog.info("taosd found in %s" % buildPath) taosBenchbin = buildPath+ "/build/bin/taosBenchmark" + buildPath = self.getBuildPath() + config = buildPath+ "../sim/dnode1/cfg/" + tsql=self.newcur(host,config) # insert: create one or mutiple tables per sql and insert multiple rows per sql - tdSql.execute("drop database if exists %s"%dbname) + tsql.execute("drop database if exists %s"%dbname) - tdSql.execute("create database %s vgroups %d"%(dbname,vgroups)) - tdSql.execute("use %s" %dbname) + tsql.execute("create database %s vgroups %d"%(dbname,vgroups)) + print("db has been created") + # tsql.getResult("show databases") + # print(tdSql.queryResult) + tsql.execute("use %s" %dbname) threads = [] # threadNumbers=2 @@ -220,7 +227,9 @@ class TDTestCase: jsonfile="1-insert/Vgroups%d%d.json"%(vgroups,i) os.system("cp -f 1-insert/manyVgroups.json %s"%(jsonfile)) os.system("sed -i 's/\"name\": \"db\",/\"name\": \"%s\",/g' %s"%(dbname,jsonfile)) - os.system("sed -i 's/\"childtable_count\": 300000,/\"childtable_count\": %d,/g' %s "%(count,jsonfile)) + os.system("sed -i 's/\"drop\": \"no\",/\"drop\": \"%s\",/g' %s"%(dropdb,jsonfile)) + os.system("sed -i 's/\"host\": \"127.0.0.1\",/\"host\": \"%s\",/g' %s"%(host,jsonfile)) + os.system("sed -i 's/\"childtable_count\": 10000,/\"childtable_count\": %d,/g' %s "%(count,jsonfile)) os.system("sed -i 's/\"name\": \"stb1\",/\"name\": \"%s%d\",/g' %s "%(stbname,i,jsonfile)) os.system("sed -i 's/\"childtable_prefix\": \"stb1_\",/\"childtable_prefix\": \"%s%d_\",/g' %s "%(stbname,i,jsonfile)) threads.append(mp.Process(target=self.taosBench, args=("%s"%jsonfile,))) @@ -337,7 +346,8 @@ class TDTestCase: return def test_case3(self): - self.taosBenchCreate("db1", "stb1", 1, 2, 1*50000) + # self.taosBenchCreate("chenhaoran02","no","db1", "stb1", 1, 8, 1*10000) + self.taosBenchCreate("chenhaoran02","no","db1", "stb1", 1, 8, 1*1000) # self.taosBenchCreate("db1", "stb1", 4, 5, 100*10000) # self.taosBenchCreate("db1", "stb1", 1, 5, 100*10000) diff --git a/tests/system-test/1-insert/manyVgroups.json b/tests/system-test/1-insert/manyVgroups.json index 6ce78af334..5487dff708 100644 --- a/tests/system-test/1-insert/manyVgroups.json +++ b/tests/system-test/1-insert/manyVgroups.json @@ -1,7 +1,7 @@ { "filetype": "insert", "cfgdir": "/etc/taos/", - "host": "test216", + "host": "127.0.0.1", "port": 6030, "user": "root", "password": "taosdata", @@ -23,7 +23,7 @@ { "name": "stb1", "child_table_exists": "no", - "childtable_count": 300000, + "childtable_count": 10000, "childtable_prefix": "stb1_", "auto_create_table": "no", "batch_create_tbl_num": 50000, From 6c844c7567dfa875ff90005c9f35561a10818d8a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 9 May 2022 23:34:48 +0800 Subject: [PATCH 22/29] enh(index): support numberic and json filter --- source/libs/index/inc/indexComm.h | 12 +++ source/libs/index/src/indexCache.c | 48 +-------- source/libs/index/src/indexComm.c | 47 +++++++++ source/libs/index/src/indexTfile.c | 149 ++++++++++++++++++++++----- source/libs/transport/src/transSrv.c | 2 +- 5 files changed, 187 insertions(+), 71 deletions(-) diff --git a/source/libs/index/inc/indexComm.h b/source/libs/index/inc/indexComm.h index 3b07429089..4cab71f92c 100644 --- a/source/libs/index/inc/indexComm.h +++ b/source/libs/index/inc/indexComm.h @@ -20,11 +20,23 @@ extern "C" { #endif +#include "indexInt.h" +#include "tcompare.h" + extern char JSON_COLUMN[]; extern char JSON_VALUE_DELIM; char* indexPackJsonData(SIndexTerm* itm); char* indexPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip); + +typedef enum { MATCH, CONTINUE, BREAK } TExeCond; + +typedef TExeCond (*_cache_range_compare)(void* a, void* b, int8_t type); + +TExeCond tDoCommpare(__compar_fn_t func, int8_t comType, void* a, void* b); + +_cache_range_compare indexGetCompare(RangeType ty); + #ifdef __cplusplus } #endif diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 929f33909e..0653c1d1fa 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -60,50 +60,6 @@ static int32_t cacheSearchRange_JSON(void* cache, SIndexTerm* ct, SIdxTempResult static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s, RangeType type); -typedef enum { MATCH, CONTINUE, BREAK } TExeCond; -typedef TExeCond (*_cache_range_compare)(void* a, void* b, int8_t type); - -static TExeCond tDoCommpare(__compar_fn_t func, int8_t comType, void* a, void* b) { - // optime later - int32_t ret = func(a, b); - switch (comType) { - case QUERY_LESS_THAN: { - if (ret < 0) return MATCH; - } break; - case QUERY_LESS_EQUAL: { - if (ret <= 0) return MATCH; - break; - } - case QUERY_GREATER_THAN: { - if (ret > 0) return MATCH; - break; - } - case QUERY_GREATER_EQUAL: { - if (ret >= 0) return MATCH; - } - } - return CONTINUE; -} -static TExeCond tCompareLessThan(void* a, void* b, int8_t type) { - __compar_fn_t func = getComparFunc(type, 0); - return tDoCommpare(func, QUERY_LESS_THAN, a, b); -} -static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) { - __compar_fn_t func = getComparFunc(type, 0); - return tDoCommpare(func, QUERY_LESS_EQUAL, a, b); -} -static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) { - __compar_fn_t func = getComparFunc(type, 0); - return tDoCommpare(func, QUERY_GREATER_THAN, a, b); -} -static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { - __compar_fn_t func = getComparFunc(type, 0); - return tDoCommpare(func, QUERY_GREATER_EQUAL, a, b); -} - -static TExeCond (*rangeCompare[])(void* a, void* b, int8_t type) = {tCompareLessThan, tCompareLessEqual, - tCompareGreaterThan, tCompareGreaterEqual}; - static int32_t (*cacheSearch[][QUERY_MAX])(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s) = { {cacheSearchTerm, cacheSearchPrefix, cacheSearchSuffix, cacheSearchRegex, cacheSearchLessThan, cacheSearchLessEqual, cacheSearchGreaterThan, cacheSearchGreaterEqual, cacheSearchRange}, @@ -169,7 +125,7 @@ static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* term, SIdxTempRes return 0; } - _cache_range_compare cmpFn = rangeCompare[type]; + _cache_range_compare cmpFn = indexGetCompare(type); MemTable* mem = cache; IndexCache* pCache = mem->pCache; @@ -295,7 +251,7 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTe if (cache == NULL) { return 0; } - _cache_range_compare cmpFn = rangeCompare[type]; + _cache_range_compare cmpFn = indexGetCompare(type); MemTable* mem = cache; IndexCache* pCache = mem->pCache; diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index cdd7b35675..9e85a6680a 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -13,12 +13,58 @@ * along with this program. If not, see . */ +#include "indexComm.h" #include "index.h" #include "indexInt.h" +#include "tcompare.h" char JSON_COLUMN[] = "JSON"; char JSON_VALUE_DELIM = '&'; +static TExeCond tCompareLessThan(void* a, void* b, int8_t type) { + __compar_fn_t func = getComparFunc(type, 0); + return tDoCommpare(func, QUERY_LESS_THAN, a, b); +} +static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) { + __compar_fn_t func = getComparFunc(type, 0); + return tDoCommpare(func, QUERY_LESS_EQUAL, a, b); +} +static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) { + __compar_fn_t func = getComparFunc(type, 0); + return tDoCommpare(func, QUERY_GREATER_THAN, a, b); +} +static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { + __compar_fn_t func = getComparFunc(type, 0); + return tDoCommpare(func, QUERY_GREATER_EQUAL, a, b); +} + +TExeCond tDoCommpare(__compar_fn_t func, int8_t comType, void* a, void* b) { + // optime later + int32_t ret = func(a, b); + switch (comType) { + case QUERY_LESS_THAN: { + if (ret < 0) return MATCH; + } break; + case QUERY_LESS_EQUAL: { + if (ret <= 0) return MATCH; + break; + } + case QUERY_GREATER_THAN: { + if (ret > 0) return MATCH; + break; + } + case QUERY_GREATER_EQUAL: { + if (ret >= 0) return MATCH; + } + } + return CONTINUE; +} + +static TExeCond (*rangeCompare[])(void* a, void* b, int8_t type) = {tCompareLessThan, tCompareLessEqual, + tCompareGreaterThan, tCompareGreaterEqual}; + +_cache_range_compare indexGetCompare(RangeType ty) { return rangeCompare[ty]; } + char* indexPackJsonData(SIndexTerm* itm) { /* * |<-----colname---->|<-----dataType---->|<--------colVal---------->| @@ -46,6 +92,7 @@ char* indexPackJsonData(SIndexTerm* itm) { return buf; } + char* indexPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip) { /* * |<-----colname---->|<-----dataType---->|<--------colVal---------->| diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index b5551e825f..c56d65fc6a 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -72,9 +72,23 @@ static int32_t tfSearchRange(void* reader, SIndexTerm* tem, SIdxTempResult* tr); static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTempResult* tr, RangeType ctype); -static int32_t (*tfSearch[])(void* reader, SIndexTerm* tem, SIdxTempResult* tr) = { - tfSearchTerm, tfSearchPrefix, tfSearchSuffix, tfSearchRegex, tfSearchLessThan, - tfSearchLessEqual, tfSearchGreaterThan, tfSearchGreaterEqual, tfSearchRange}; +static int32_t tfSearchTerm_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchPrefix_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchSuffix_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchRegex_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchLessThan_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchLessEqual_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchGreaterThan_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchGreaterEqual_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchRange_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr); + +static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr, RangeType ctype); + +static int32_t (*tfSearch[][QUERY_MAX])(void* reader, SIndexTerm* tem, SIdxTempResult* tr) = { + {tfSearchTerm, tfSearchPrefix, tfSearchSuffix, tfSearchRegex, tfSearchLessThan, tfSearchLessEqual, + tfSearchGreaterThan, tfSearchGreaterEqual, tfSearchRange}, + {tfSearchTerm_JSON, tfSearchPrefix_JSON, tfSearchSuffix_JSON, tfSearchRegex_JSON, tfSearchLessThan_JSON, + tfSearchLessEqual_JSON, tfSearchGreaterThan_JSON, tfSearchGreaterEqual_JSON, tfSearchRange_JSON}}; TFileCache* tfileCacheCreate(const char* path) { TFileCache* tcache = taosMemoryCalloc(1, sizeof(TFileCache)); @@ -202,14 +216,10 @@ void tfileReaderDestroy(TFileReader* reader) { taosMemoryFree(reader); } static int32_t tfSearchTerm(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { - bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON); int ret = 0; char* p = tem->colVal; uint64_t sz = tem->nColVal; - if (hasJson) { - p = indexPackJsonData(tem); - sz = strlen(p); - } + int64_t st = taosGetTimestampUs(); FstSlice key = fstSliceCreate(p, sz); uint64_t offset; @@ -224,9 +234,6 @@ static int32_t tfSearchTerm(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, load all table info, time cost: %" PRIu64 "us", tem->suid, tem->colName, tem->colVal, cost); } - if (hasJson) { - taosMemoryFree(p); - } fstSliceDestroy(&key); return 0; } @@ -308,14 +315,11 @@ static int32_t tfSearchRegex(void* reader, SIndexTerm* tem, SIdxTempResult* tr) } static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTempResult* tr, RangeType type) { - bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON); - int ret = 0; - char* p = tem->colVal; - int skip = 0; + int ret = 0; + char* p = tem->colVal; + int skip = 0; + _cache_range_compare cmpFn = indexGetCompare(type); - if (hasJson) { - p = indexPackJsonDataPrefix(tem, &skip); - } SArray* offsets = taosArrayInit(16, sizeof(uint64_t)); AutomationCtx* ctx = automCtxCreate((void*)p, AUTOMATION_ALWAYS); @@ -328,7 +332,16 @@ static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTempResult StreamWithState* st = streamBuilderIntoStream(sb); StreamWithStateResult* rt = NULL; while ((rt = streamWithStateNextWith(st, NULL)) != NULL) { - taosArrayPush(offsets, &(rt->out.out)); + FstSlice* s = &rt->data; + char* ch = (char*)fstSliceData(s, NULL); + TExeCond cond = cmpFn(ch, p, tem->colType); + if (MATCH == cond) { + tfileReaderLoadTableIds((TFileReader*)reader, rt->out.out, tr->total); + } else if (CONTINUE == cond) { + } else if (BREAK == cond) { + swsResultDestroy(rt); + break; + } swsResultDestroy(rt); } streamWithStateDestroy(st); @@ -376,17 +389,105 @@ static int32_t tfSearchRange(void* reader, SIndexTerm* tem, SIdxTempResult* tr) fstSliceDestroy(&key); return 0; } +static int32_t tfSearchTerm_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + int ret = 0; + char* p = indexPackJsonData(tem); + int sz = strlen(p); + int64_t st = taosGetTimestampUs(); + FstSlice key = fstSliceCreate(p, sz); + uint64_t offset; + if (fstGet(((TFileReader*)reader)->fst, &key, &offset)) { + int64_t et = taosGetTimestampUs(); + int64_t cost = et - st; + indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, found table info in tindex, time cost: %" PRIu64 "us", + tem->suid, tem->colName, tem->colVal, cost); + + ret = tfileReaderLoadTableIds((TFileReader*)reader, offset, tr->total); + cost = taosGetTimestampUs() - et; + indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, load all table info, time cost: %" PRIu64 "us", tem->suid, + tem->colName, tem->colVal, cost); + } + fstSliceDestroy(&key); + return 0; + // deprecate api + return TSDB_CODE_SUCCESS; +} +static int32_t tfSearchPrefix_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + // impl later + return TSDB_CODE_SUCCESS; +} +static int32_t tfSearchSuffix_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + // impl later + return TSDB_CODE_SUCCESS; +} +static int32_t tfSearchRegex_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + // impl later + return TSDB_CODE_SUCCESS; +} +static int32_t tfSearchLessThan_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + return tfSearchCompareFunc_JSON(reader, tem, tr, LT); +} +static int32_t tfSearchLessEqual_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + return tfSearchCompareFunc_JSON(reader, tem, tr, LE); +} +static int32_t tfSearchGreaterThan_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + return tfSearchCompareFunc_JSON(reader, tem, tr, GT); +} +static int32_t tfSearchGreaterEqual_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + return tfSearchCompareFunc_JSON(reader, tem, tr, GE); +} +static int32_t tfSearchRange_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + // impl later + return TSDB_CODE_SUCCESS; +} + +static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr, RangeType ctype) { + int ret = 0; + int skip = 0; + + char* p = indexPackJsonDataPrefix(tem, &skip); + + _cache_range_compare cmpFn = indexGetCompare(ctype); + + SArray* offsets = taosArrayInit(16, sizeof(uint64_t)); + + AutomationCtx* ctx = automCtxCreate((void*)p, AUTOMATION_PREFIX); + FstStreamBuilder* sb = fstSearch(((TFileReader*)reader)->fst, ctx); + + FstSlice h = fstSliceCreate((uint8_t*)p, skip); + fstStreamBuilderSetRange(sb, &h, ctype); + fstSliceDestroy(&h); + + StreamWithState* st = streamBuilderIntoStream(sb); + StreamWithStateResult* rt = NULL; + while ((rt = streamWithStateNextWith(st, NULL)) != NULL) { + FstSlice* s = &rt->data; + char* ch = (char*)fstSliceData(s, NULL); + TExeCond cond = cmpFn(ch, p, tem->colType); + if (MATCH == cond) { + tfileReaderLoadTableIds((TFileReader*)reader, rt->out.out, tr->total); + } else if (CONTINUE == cond) { + } else if (BREAK == cond) { + swsResultDestroy(rt); + break; + } + swsResultDestroy(rt); + } + streamWithStateDestroy(st); + fstStreamBuilderDestroy(sb); + return TSDB_CODE_SUCCESS; +} int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SIdxTempResult* tr) { SIndexTerm* term = query->term; EIndexQueryType qtype = query->qType; - if (qtype >= sizeof(tfSearch) / sizeof(tfSearch[0])) { - indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, not found table info in tindex", term->suid, term->colName, - term->colVal); - return -1; + + if (INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) { + return tfSearch[1][qtype](reader, term, tr); } else { - return tfSearch[qtype](reader, term, tr); + return tfSearch[0][qtype](reader, term, tr); } + tfileReaderUnRef(reader); return 0; } diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index 27efbcda53..f6fa7b93fd 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -1070,7 +1070,7 @@ void transSendResponse(const STransMsg* msg) { SSrvMsg* srvMsg = taosMemoryCalloc(1, sizeof(SSrvMsg)); srvMsg->msg = tmsg; srvMsg->type = Normal; - tTrace("server conn %p start to send resp (1/2)", exh->handle); + tDebug("server conn %p start to send resp (1/2)", exh->handle); transSendAsync(pThrd->asyncPool, &srvMsg->q); uvReleaseExHandle(refId); return; From 60efd600b8acce556ca5c3488bd384608afc5955 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 10 May 2022 01:57:08 +0000 Subject: [PATCH 23/29] refact --- include/common/trow.h | 41 ++++++++++++++++------------------------- 1 file changed, 16 insertions(+), 25 deletions(-) diff --git a/include/common/trow.h b/include/common/trow.h index eed15689c0..4031946ee8 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -30,6 +30,22 @@ extern "C" { #endif +typedef struct { + TSKEY ts; + union { + uint32_t info; + struct { + uint16_t type : 2; + uint16_t del : 1; + uint16_t endian : 1; + uint16_t reserve : 12; + uint16_t sver; + }; + }; + uint32_t len; + char data[]; +} STSRow; + // Target of tdataformat.h: // 1. Row related definition in dataformat.h of 2.0 could be replaced with tdataformat.h of 3.0. // 2. The basic definition in dataformat.h is shared with tdataformat.h of 3.0. @@ -117,31 +133,6 @@ typedef struct { SKvRowIdx cidx[]; } SKvRow; -typedef struct { - /// timestamp - TSKEY ts; - union { - /// union field for encode and decode - uint32_t info; - struct { - /// row type - uint16_t type : 2; - /// is delete row(0 not delete, 1 delete) - uint16_t del : 1; - /// endian(0 little endian, 1 big endian) - uint16_t endian : 1; - /// reserved for back compatibility - uint16_t reserve : 12; - /// row schema version - uint16_t sver; - }; - }; - /// row total length - uint32_t len; - /// the inline data, maybe a tuple or a k-v tuple - char data[]; -} STSRow; - typedef struct { // basic info int8_t rowType; From baddf3207659ea9645e47349173a456ca74779c4 Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Tue, 10 May 2022 10:15:16 +0800 Subject: [PATCH 24/29] update test case --- tests/system-test/2-query/diff.py | 49 ++++++++----------------------- 1 file changed, 12 insertions(+), 37 deletions(-) diff --git a/tests/system-test/2-query/diff.py b/tests/system-test/2-query/diff.py index 1a6a2e7cf1..03b3899dc6 100644 --- a/tests/system-test/2-query/diff.py +++ b/tests/system-test/2-query/diff.py @@ -83,17 +83,20 @@ class TDTestCase: tdSql.query("select diff(col6) from stb_1") tdSql.checkRows(0) + tdSql.query("select diff(col7) from stb_1") + tdSql.checkRows(0) + for i in range(self.rowNum): tdSql.execute("insert into stb_1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) - tdSql.error("select diff(ts) from stb") + # tdSql.error("select diff(ts) from stb") tdSql.error("select diff(ts) from stb_1") - tdSql.error("select diff(col7) from stb") - tdSql.error("select diff(col7) from stb_1") - tdSql.error("select diff(col8) from stb") + # tdSql.error("select diff(col7) from stb") + + # tdSql.error("select diff(col8) from stb") tdSql.error("select diff(col8) from stb_1") - tdSql.error("select diff(col9) from stb") + # tdSql.error("select diff(col9) from stb") tdSql.error("select diff(col9) from stb_1") tdSql.error("select diff(col11) from stb_1") tdSql.error("select diff(col12) from stb_1") @@ -101,31 +104,12 @@ class TDTestCase: tdSql.error("select diff(col14) from stb_1") tdSql.query("select ts,diff(col1),ts from stb_1") - tdSql.checkRows(10) + tdSql.checkRows(11) tdSql.checkData(0, 0, "2018-09-17 09:00:00.000") - tdSql.checkData(0, 1, "2018-09-17 09:00:00.000") - tdSql.checkData(0, 3, "2018-09-17 09:00:00.000") + tdSql.checkData(1, 0, "2018-09-17 09:00:00.000") + tdSql.checkData(1, 2, "2018-09-17 09:00:00.000") tdSql.checkData(9, 0, "2018-09-17 09:00:00.009") - tdSql.checkData(9, 1, "2018-09-17 09:00:00.009") - tdSql.checkData(9, 3, "2018-09-17 09:00:00.009") - - # tdSql.query("select ts,diff(col1),ts from stb group by tbname") - # tdSql.checkRows(10) - # tdSql.checkData(0, 0, "2018-09-17 09:00:00.000") - # tdSql.checkData(0, 1, "2018-09-17 09:00:00.000") - # tdSql.checkData(0, 3, "2018-09-17 09:00:00.000") - # tdSql.checkData(9, 0, "2018-09-17 09:00:00.009") - # tdSql.checkData(9, 1, "2018-09-17 09:00:00.009") - # tdSql.checkData(9, 3, "2018-09-17 09:00:00.009") - - tdSql.query("select ts,diff(col1),ts from stb_1") - tdSql.checkRows(10) - tdSql.checkData(0, 0, "2018-09-17 09:00:00.000") - tdSql.checkData(0, 1, "2018-09-17 09:00:00.000") - tdSql.checkData(0, 3, "2018-09-17 09:00:00.000") - tdSql.checkData(9, 0, "2018-09-17 09:00:00.009") - tdSql.checkData(9, 1, "2018-09-17 09:00:00.009") - tdSql.checkData(9, 3, "2018-09-17 09:00:00.009") + tdSql.checkData(9, 2, "2018-09-17 09:00:00.009") # tdSql.query("select ts,diff(col1),ts from stb group by tbname") # tdSql.checkRows(10) @@ -153,15 +137,6 @@ class TDTestCase: tdSql.query("select diff(col6) from stb_1") tdSql.checkRows(10) - - self.insertData() - - tdSql.query("select diff(col) from st group by tbname") - tdSql.checkRows(185) - - tdSql.error("select diff(col) from st group by dev") - - tdSql.error("select diff(col) from st group by col") def stop(self): tdSql.close() From 72135488478ac1640faba440b01443c23f4c98cb Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 9 May 2022 23:09:24 +0800 Subject: [PATCH 25/29] fix(query): fix histogram normalized result incorrect for super table TD-15407 --- source/libs/function/src/builtinsimpl.c | 27 ++++++++++++++----------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 75efb533ad..009b964eb9 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -110,6 +110,7 @@ typedef struct SHistoFuncBin { typedef struct SHistoFuncInfo { int32_t numOfBins; + int32_t totalCount; bool normalized; SHistoFuncBin bins[]; } SHistoFuncInfo; @@ -2289,6 +2290,9 @@ bool histogramFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo *pResultIn } SHistoFuncInfo *pInfo = GET_ROWCELL_INTERBUF(pResultInfo); + pInfo->numOfBins = 0; + pInfo->totalCount = 0; + pInfo->normalized = 0; int8_t binType = getHistogramBinType(varDataVal(pCtx->param[1].param.pz)); if (binType == UNKNOWN_BIN) { @@ -2318,7 +2322,6 @@ int32_t histogramFunction(SqlFunctionCtx *pCtx) { int32_t numOfRows = pInput->numOfRows; int32_t numOfElems = 0; - int32_t totalElems = 0; for (int32_t i = start; i < numOfRows + start; ++i) { if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { continue; @@ -2333,23 +2336,13 @@ int32_t histogramFunction(SqlFunctionCtx *pCtx) { for (int32_t k = 0; k < pInfo->numOfBins; ++k) { if (v > pInfo->bins[k].lower && v <= pInfo->bins[k].upper) { pInfo->bins[k].count++; - totalElems++; + pInfo->totalCount++; break; } } } - if (pInfo->normalized) { - for (int32_t k = 0; k < pInfo->numOfBins; ++k) { - if(totalElems != 0) { - pInfo->bins[k].percentage = pInfo->bins[k].count / (double)totalElems; - } else { - pInfo->bins[k].percentage = 0; - } - } - } - SET_VAL(GET_RES_INFO(pCtx), numOfElems, pInfo->numOfBins); return TSDB_CODE_SUCCESS; } @@ -2362,6 +2355,16 @@ int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t currentRow = pBlock->info.rows; + if (pInfo->normalized) { + for (int32_t k = 0; k < pResInfo->numOfRes; ++k) { + if(pInfo->totalCount != 0) { + pInfo->bins[k].percentage = pInfo->bins[k].count / (double)pInfo->totalCount; + } else { + pInfo->bins[k].percentage = 0; + } + } + } + for (int32_t i = 0; i < pResInfo->numOfRes; ++i) { int32_t len; char buf[512] = {0}; From 728d63c6646923f3feed8f5dcf47a70156280c50 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 10 May 2022 12:19:32 +0800 Subject: [PATCH 26/29] ehn: remove debug log --- include/common/taosdef.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/common/taosdef.h b/include/common/taosdef.h index 5384082da3..e1f8832edf 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -89,7 +89,7 @@ extern char *qtypeStr[]; #define TSDB_PORT_HTTP 11 -#define TD_DEBUG_PRINT_ROW +#undef TD_DEBUG_PRINT_ROW #ifdef __cplusplus } From 8c0d600cdf2dff21c9bd5fea17d5e44d4c1ec0a2 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 10 May 2022 12:20:13 +0800 Subject: [PATCH 27/29] enh(sync): add error log --- source/libs/sync/src/syncRaftLog.c | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 031722ab3c..a422ddf4db 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -60,7 +60,7 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { if (code != 0) { int32_t err = terrno; const char *errStr = tstrerror(err); - sError("walWriteWithSyncInfo error, err:%d, msg:%s", err, errStr); + sError("walWriteWithSyncInfo error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, errno, strerror(errno)); ASSERT(0); } //assert(code == 0); @@ -79,7 +79,7 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { if (code != 0) { int32_t err = terrno; const char *errStr = tstrerror(err); - sError("walReadWithHandle error, err:%d, msg:%s", err, errStr); + sError("walWriteWithSyncInfo error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, errno, strerror(errno)); ASSERT(0); } //assert(walReadWithHandle(pWalHandle, index) == 0); @@ -108,7 +108,14 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; - assert(walRollback(pWal, fromIndex) == 0); + //assert(walRollback(pWal, fromIndex) == 0); + int32_t code = walRollback(pWal, fromIndex); + if (code != 0) { + int32_t err = terrno; + const char *errStr = tstrerror(err); + sError("walRollback error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, errno, strerror(errno)); + ASSERT(0); + } return 0; // to avoid compiler error } @@ -132,7 +139,14 @@ SyncTerm logStoreLastTerm(SSyncLogStore* pLogStore) { int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; - assert(walCommit(pWal, index) == 0); + //assert(walCommit(pWal, index) == 0); + int32_t code = walCommit(pWal, index); + if (code != 0) { + int32_t err = terrno; + const char *errStr = tstrerror(err); + sError("walCommit error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, errno, strerror(errno)); + ASSERT(0); + } return 0; // to avoid compiler error } From 5a8497fe4568b3f4282b2f25163bc1612c40d5ed Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 10 May 2022 12:21:47 +0800 Subject: [PATCH 28/29] fix: undef debug flag --- include/common/taosdef.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/common/taosdef.h b/include/common/taosdef.h index 5384082da3..e1f8832edf 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -89,7 +89,7 @@ extern char *qtypeStr[]; #define TSDB_PORT_HTTP 11 -#define TD_DEBUG_PRINT_ROW +#undef TD_DEBUG_PRINT_ROW #ifdef __cplusplus } From 503de649cbed621b4585b4926741535bfd2b4f62 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 10 May 2022 12:22:57 +0800 Subject: [PATCH 29/29] enh(sync): add error log --- source/libs/sync/src/syncRaftLog.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index a422ddf4db..66c85b9cbb 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -79,7 +79,7 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { if (code != 0) { int32_t err = terrno; const char *errStr = tstrerror(err); - sError("walWriteWithSyncInfo error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, errno, strerror(errno)); + sError("walReadWithHandle error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, errno, strerror(errno)); ASSERT(0); } //assert(walReadWithHandle(pWalHandle, index) == 0);