From c3484ce8c372a14031476e8b8b7318f532a18487 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 29 Apr 2022 20:04:34 +0800 Subject: [PATCH 01/48] stmt query --- source/client/src/clientStmt.c | 18 +- source/libs/scalar/src/sclvector.c | 22 +- tests/script/api/batchprepare.c | 1100 ++-------------------------- 3 files changed, 86 insertions(+), 1054 deletions(-) diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 5ddffa0cbd..f7ef1f7e81 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -649,6 +649,19 @@ int stmtGetParamNum(TAOS_STMT *stmt, int *nums) { STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS)); + if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { + pStmt->bInfo.needParse = false; + } + + if (pStmt->exec.pRequest && STMT_TYPE_QUERY == pStmt->sql.type && pStmt->sql.runTimes) { + taos_free_result(pStmt->exec.pRequest); + pStmt->exec.pRequest = NULL; + } + + if (NULL == pStmt->exec.pRequest) { + STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest)); + } + if (pStmt->bInfo.needParse) { STMT_ERR_RET(stmtParseSql(pStmt)); } @@ -658,8 +671,11 @@ int stmtGetParamNum(TAOS_STMT *stmt, int *nums) { STMT_ERR_RET(getQueryPlan(pStmt->exec.pRequest, pStmt->sql.pQuery, &pStmt->sql.nodeList)); pStmt->sql.pQueryPlan = pStmt->exec.pRequest->body.pDag; pStmt->exec.pRequest->body.pDag = NULL; + STMT_ERR_RET(stmtBackupQueryFields(pStmt)); + } else { + STMT_ERR_RET(stmtRestoreQueryFields(pStmt)); } - + *nums = taosArrayGetSize(pStmt->sql.pQueryPlan->pPlaceholderValues); } else { STMT_ERR_RET(stmtFetchColFields(stmt, nums, NULL)); diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index a75b2521bd..84aa5559d0 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -297,6 +297,22 @@ static FORCE_INLINE void varToNchar(char* buf, SScalarParam* pOut, int32_t rowIn taosMemoryFree(t); } +static FORCE_INLINE void ncharToVar(char* buf, SScalarParam* pOut, int32_t rowIndex) { + int32_t inputLen = varDataLen(buf); + + char* t = taosMemoryCalloc(1, inputLen + VARSTR_HEADER_SIZE); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(buf), varDataLen(buf), varDataVal(t)); + if (len < 0) { + taosMemoryFree(t); + return; + } + varDataSetLen(t, len); + + colDataAppend(pOut->columnData, rowIndex, t, false); + taosMemoryFree(t); +} + + //TODO opt performance, tmp is not needed. int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, int32_t inType, int32_t outType) { int32_t bufSize = pIn->columnData->info.bytes; @@ -313,6 +329,10 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in func = varToUnsigned; } else if (IS_FLOAT_TYPE(outType)) { func = varToFloat; + } else if (outType == TSDB_DATA_TYPE_BINARY) { // nchar -> binary + ASSERT(inType == TSDB_DATA_TYPE_NCHAR); + func = ncharToVar; + vton = true; } else if (outType == TSDB_DATA_TYPE_NCHAR) { // binary -> nchar ASSERT(inType == TSDB_DATA_TYPE_VARCHAR); func = varToNchar; @@ -608,7 +628,7 @@ int8_t gConvertTypes[TSDB_DATA_TYPE_BLOB+1][TSDB_DATA_TYPE_BLOB+1] = { /*BIGI*/ 0, 0, 0, 0, 0, 0, 6, 7, 7, 0, 7, 5, 5, 5, 7, 0, 7, 0, 0, /*FLOA*/ 0, 0, 0, 0, 0, 0, 0, 7, 7, 6, 7, 6, 6, 6, 6, 0, 7, 0, 0, /*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 0, 7, 0, 0, -/*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 7, 7, 7, 7, 0, 0, 0, 0, +/*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8, 7, 7, 7, 7, 0, 0, 0, 0, /*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 7, 0, 7, 0, 0, /*NCHA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 0, 0, 0, 0, /*UTIN*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 0, 7, 0, 0, diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index f14885b72e..7bea19ae90 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -11,8 +11,8 @@ int32_t shortColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_INT}; int32_t fullColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_BOOL, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_UTINYINT, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_USMALLINT, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_UINT, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_UBIGINT, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_BINARY, TSDB_DATA_TYPE_NCHAR}; -int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_NCHAR, TSDB_DATA_TYPE_BOOL}; -int32_t optrIdxList[] = {2, 11, 6}; +int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_DOUBLE}; +int32_t optrIdxList[] = {0, 1, 2}; typedef struct { char* oper; @@ -169,8 +169,8 @@ typedef struct { int32_t caseRunNum; // total run case num } CaseCtrl; -#if 1 -CaseCtrl gCaseCtrl = { +#if 0 +CaseCtrl gCaseCtrl = { // default .bindNullNum = 0, .prepareStb = false, .printCreateTblSql = false, @@ -186,23 +186,15 @@ CaseCtrl gCaseCtrl = { .checkParamNum = false, .printRes = true, .runTimes = 0, -// .caseIdx = -1, -// .caseNum = -1, + .caseIdx = -1, + .caseNum = -1, .caseRunIdx = -1, -// .caseRunNum = -1, - - -// .optrIdxListNum = tListLen(optrIdxList), -// .optrIdxList = optrIdxList, -// .bindColTypeNum = tListLen(bindColTypeList), -// .bindColTypeList = bindColTypeList, - .caseIdx = 22, - .caseNum = 1, - .caseRunNum = 1, - + .caseRunNum = -1, }; -#else -CaseCtrl gCaseCtrl = { +#endif + +#if 0 +CaseCtrl gCaseCtrl = { // query case with specified col&oper .bindNullNum = 0, .prepareStb = false, .printCreateTblSql = false, @@ -211,17 +203,50 @@ CaseCtrl gCaseCtrl = { .rowNum = 0, .bindColNum = 0, .bindRowNum = 0, - .bindColTypeNum = tListLen(bindColTypeList), - .bindColTypeList = bindColTypeList, + .bindColTypeNum = 0, + .bindColTypeList = NULL, + .optrIdxListNum = 0, + .optrIdxList = NULL, .checkParamNum = false, .printRes = true, .runTimes = 0, - .caseIdx = 2, - .caseNum = 1, .caseRunIdx = -1, + .optrIdxListNum = 0, + .optrIdxList = NULL, + .bindColTypeNum = 0, + .bindColTypeList = NULL, + .caseIdx = 22, + .caseNum = 1, .caseRunNum = 1, }; +#endif +#if 1 +CaseCtrl gCaseCtrl = { // query case with specified col&oper + .bindNullNum = 0, + .prepareStb = false, + .printCreateTblSql = false, + .printQuerySql = true, + .printStmtSql = true, + .rowNum = 0, + .bindColNum = 0, + .bindRowNum = 0, + .bindColTypeNum = 0, + .bindColTypeList = NULL, + .optrIdxListNum = 0, + .optrIdxList = NULL, + .checkParamNum = true, + .printRes = true, + .runTimes = 0, + .caseRunIdx = -1, + .optrIdxListNum = tListLen(optrIdxList), + .optrIdxList = optrIdxList, + .bindColTypeNum = tListLen(bindColTypeList), + .bindColTypeList = bindColTypeList, + .caseIdx = 22, + .caseNum = 1, + .caseRunNum = 1, +}; #endif int32_t taosGetTimeOfDay(struct timeval *tv) { @@ -3218,1035 +3243,6 @@ int stmt_funcb_autoctb_e5(TAOS_STMT *stmt) { return 0; } -//samets -int stmt_funcb4(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[60]; - int8_t v1[60]; - int16_t v2[60]; - int32_t v4[60]; - int64_t v8[60]; - float f4[60]; - double f8[60]; - char bin[60][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - - int *lb = taosMemoryMalloc(60 * sizeof(int)); - - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = taosMemoryMalloc(sizeof(char) * 60); - char* no_null = taosMemoryMalloc(sizeof(char) * 60); - - for (int i = 0; i < 60; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 9000000; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[60*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 60; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 60; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 60; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 60; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 60; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 60; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 60; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 60; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 60; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 60; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 54000000; ++i) { - v.ts[i] = tts; - } - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - int id = 0; - for (int l = 0; l < 3000; l++) { - for (int zz = 0; zz < 300; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - - taos_stmt_bind_param_batch(stmt, params + id * 10); - taos_stmt_add_batch(stmt); - } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - ++id; - } - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - - return 0; -} - - - - -//1table 18000 reocrds -int stmt_funcb5(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[18000]; - int8_t v1[18000]; - int16_t v2[18000]; - int32_t v4[18000]; - int64_t v8[18000]; - float f4[18000]; - double f8[18000]; - char bin[18000][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - - int *lb = taosMemoryMalloc(18000 * sizeof(int)); - - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); - char* is_null = taosMemoryMalloc(sizeof(char) * 18000); - char* no_null = taosMemoryMalloc(sizeof(char) * 18000); - - for (int i = 0; i < 18000; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 30000; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[18000*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 18000; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 18000; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 18000; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 18000; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 18000; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 18000; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 18000; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 18000; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 18000; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 18000; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 54000000; ++i) { - v.ts[i] = tts + i; - } - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into m0 values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - int id = 0; - for (int l = 0; l < 10; l++) { - for (int zz = 0; zz < 1; zz++) { - taos_stmt_bind_param_batch(stmt, params + id * 10); - taos_stmt_add_batch(stmt); - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - ++id; - - } - - } - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - - return 0; -} - - -//1table 200000 reocrds -int stmt_funcb_ssz1(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int b[30000]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 30000 * 3000); - - int *lb = taosMemoryMalloc(30000 * sizeof(int)); - - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); - char* no_null = taosMemoryMalloc(sizeof(int) * 200000); - - for (int i = 0; i < 30000; ++i) { - lb[i] = 40; - no_null[i] = 0; - v.b[i] = (int8_t)(i % 2); - } - - for (int i = 0; i < 30000; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[30000*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 30000; - - params[i+1].buffer_type = TSDB_DATA_TYPE_INT; - params[i+1].buffer_length = sizeof(int); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = no_null; - params[i+1].num = 30000; - } - - int64_t tts = 0; - for (int64_t i = 0; i < 90000000LL; ++i) { - v.ts[i] = tts + i; - } - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? values(?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - int id = 0; - for (int l = 0; l < 10; l++) { - for (int zz = 0; zz < 300; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - - taos_stmt_bind_param_batch(stmt, params + id * 10); - taos_stmt_add_batch(stmt); - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - ++id; - - } - - } - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(no_null); - - return 0; -} - - -//one table 60 records one time -int stmt_funcb_s1(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[60]; - int8_t v1[60]; - int16_t v2[60]; - int32_t v4[60]; - int64_t v8[60]; - float f4[60]; - double f8[60]; - char bin[60][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - - int *lb = taosMemoryMalloc(60 * sizeof(int)); - - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = taosMemoryMalloc(sizeof(char) * 60); - char* no_null = taosMemoryMalloc(sizeof(char) * 60); - - for (int i = 0; i < 60; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 9000000; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[60*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 60; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 60; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 60; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 60; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 60; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 60; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 60; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 60; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 60; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 60; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 54000000; ++i) { - v.ts[i] = tts + i; - } - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - int id = 0; - for (int l = 0; l < 3000; l++) { - for (int zz = 0; zz < 300; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - - taos_stmt_bind_param_batch(stmt, params + id * 10); - taos_stmt_add_batch(stmt); - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - ++id; - } - - } - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - - return 0; -} - - - - - - -//300 tables 60 records single column bind -int stmt_funcb_sc1(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[60]; - int8_t v1[60]; - int16_t v2[60]; - int32_t v4[60]; - int64_t v8[60]; - float f4[60]; - double f8[60]; - char bin[60][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - - int *lb = taosMemoryMalloc(60 * sizeof(int)); - - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = taosMemoryMalloc(sizeof(char) * 60); - char* no_null = taosMemoryMalloc(sizeof(char) * 60); - - for (int i = 0; i < 60; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 9000000; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[60*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 60; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 60; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 60; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 60; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 60; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 60; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 60; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 60; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 60; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 60; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 54000000; ++i) { - v.ts[i] = tts + i; - } - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - int id = 0; - for (int l = 0; l < 3000; l++) { - for (int zz = 0; zz < 300; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - - for (int col=0; col < 10; ++col) { - taos_stmt_bind_single_param_batch(stmt, params + id++, col); - } - - taos_stmt_add_batch(stmt); - } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - } - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - - return 0; -} - - -//1 tables 60 records single column bind -int stmt_funcb_sc2(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[60]; - int8_t v1[60]; - int16_t v2[60]; - int32_t v4[60]; - int64_t v8[60]; - float f4[60]; - double f8[60]; - char bin[60][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - - int *lb = taosMemoryMalloc(60 * sizeof(int)); - - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = taosMemoryMalloc(sizeof(char) * 60); - char* no_null = taosMemoryMalloc(sizeof(char) * 60); - - for (int i = 0; i < 60; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 9000000; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[60*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 60; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 60; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 60; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 60; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 60; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 60; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 60; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 60; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 60; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 60; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 54000000; ++i) { - v.ts[i] = tts + i; - } - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - int id = 0; - for (int l = 0; l < 3000; l++) { - for (int zz = 0; zz < 300; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - - for (int col=0; col < 10; ++col) { - taos_stmt_bind_single_param_batch(stmt, params + id++, col); - } - - taos_stmt_add_batch(stmt); - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - } - - } - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - - return 0; -} - - -//10 tables [1...10] records single column bind -int stmt_funcb_sc3(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[60]; - int8_t v1[60]; - int16_t v2[60]; - int32_t v4[60]; - int64_t v8[60]; - float f4[60]; - double f8[60]; - char bin[60][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 60); - - int *lb = taosMemoryMalloc(60 * sizeof(int)); - - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 60*10); - char* is_null = taosMemoryMalloc(sizeof(char) * 60); - char* no_null = taosMemoryMalloc(sizeof(char) * 60); - - for (int i = 0; i < 60; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - int g = 0; - for (int i = 0; i < 600; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = g%10+1; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = g%10+1; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = g%10+1; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = g%10+1; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = g%10+1; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = g%10+1; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = g%10+1; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = g%10+1; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = g%10+1; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = g%10+1; - ++g; - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 60; ++i) { - v.ts[i] = tts + i; - } - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - int id = 0; - for (int zz = 0; zz < 10; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - - for (int col=0; col < 10; ++col) { - taos_stmt_bind_single_param_batch(stmt, params + id++, col); - } - - taos_stmt_add_batch(stmt); - } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - - return 0; -} #endif From 5d4a77dc80cd42e28101ff3adc4568b836a0f5f0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 29 Apr 2022 21:23:36 +0800 Subject: [PATCH 02/48] fix: invalid write in multi-process mode --- source/dnode/mgmt/implement/src/dmExec.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/mgmt/implement/src/dmExec.c b/source/dnode/mgmt/implement/src/dmExec.c index 06001028b5..bb0206d229 100644 --- a/source/dnode/mgmt/implement/src/dmExec.c +++ b/source/dnode/mgmt/implement/src/dmExec.c @@ -192,15 +192,15 @@ void dmCloseNode(SMgmtWrapper *pWrapper) { dmStopNode(pWrapper); - taosWLockLatch(&pWrapper->latch); - (*pWrapper->fp.closeFp)(pWrapper); - taosWUnLockLatch(&pWrapper->latch); - if (pWrapper->procObj) { taosProcCleanup(pWrapper->procObj); pWrapper->procObj = NULL; } + taosWLockLatch(&pWrapper->latch); + (*pWrapper->fp.closeFp)(pWrapper); + taosWUnLockLatch(&pWrapper->latch); + dInfo("node:%s, has been closed", pWrapper->name); } From 53720052eddb240889f7fd152596129e98c9aaf8 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 29 Apr 2022 21:48:28 +0800 Subject: [PATCH 03/48] fix: invalid write in multi-process mode --- source/dnode/mgmt/implement/src/dmExec.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/mgmt/implement/src/dmExec.c b/source/dnode/mgmt/implement/src/dmExec.c index bb0206d229..06001028b5 100644 --- a/source/dnode/mgmt/implement/src/dmExec.c +++ b/source/dnode/mgmt/implement/src/dmExec.c @@ -192,15 +192,15 @@ void dmCloseNode(SMgmtWrapper *pWrapper) { dmStopNode(pWrapper); + taosWLockLatch(&pWrapper->latch); + (*pWrapper->fp.closeFp)(pWrapper); + taosWUnLockLatch(&pWrapper->latch); + if (pWrapper->procObj) { taosProcCleanup(pWrapper->procObj); pWrapper->procObj = NULL; } - taosWLockLatch(&pWrapper->latch); - (*pWrapper->fp.closeFp)(pWrapper); - taosWUnLockLatch(&pWrapper->latch); - dInfo("node:%s, has been closed", pWrapper->name); } From ee5b67a965c4d30ce9415193050273f0f61bf1f7 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 1 May 2022 22:20:24 +0800 Subject: [PATCH 04/48] enh(index): support index filter --- source/libs/executor/CMakeLists.txt | 4 +- source/libs/executor/src/indexoperator.c | 181 ++++++++++++++++------- source/libs/index/src/index.c | 35 ++--- 3 files changed, 147 insertions(+), 73 deletions(-) diff --git a/source/libs/executor/CMakeLists.txt b/source/libs/executor/CMakeLists.txt index 3dfef9b59f..bfa54be71f 100644 --- a/source/libs/executor/CMakeLists.txt +++ b/source/libs/executor/CMakeLists.txt @@ -8,7 +8,7 @@ add_library(executor STATIC ${EXECUTOR_SRC}) # ) target_link_libraries(executor - PRIVATE os util common function parser planner qcom vnode scalar nodes + PRIVATE os util common function parser planner qcom vnode scalar nodes index ) target_include_directories( @@ -19,4 +19,4 @@ target_include_directories( #if(${BUILD_TEST}) ADD_SUBDIRECTORY(test) -#endif(${BUILD_TEST}) \ No newline at end of file +#endif(${BUILD_TEST}) diff --git a/source/libs/executor/src/indexoperator.c b/source/libs/executor/src/indexoperator.c index 1f8e73b80a..911ba22fe6 100644 --- a/source/libs/executor/src/indexoperator.c +++ b/source/libs/executor/src/indexoperator.c @@ -15,7 +15,9 @@ #include "indexoperator.h" #include "executorimpl.h" +#include "index.h" #include "nodes.h" +#include "tdatablock.h" typedef struct SIFCtx { int32_t code; @@ -48,11 +50,19 @@ typedef struct SIFCtx { } while (0) typedef struct SIFParam { - SArray * result; SHashObj *pFilter; + + SArray *result; + char * condValue; + + col_id_t colId; + + int64_t suid; // add later + char dbName[TSDB_DB_NAME_LEN]; + char colName[TSDB_COL_NAME_LEN]; } SIFParam; -typedef int32_t (*sif_func_t)(SNode *left, SNode *rigth, SIFParam *output); +typedef int32_t (*sif_func_t)(SIFParam *left, SIFParam *rigth, SIFParam *output); // construct tag filter operator later static void destroyTagFilterOperatorInfo(void *param) { STagFilterOperatorInfo *pInfo = (STagFilterOperatorInfo *)param; @@ -60,7 +70,10 @@ static void destroyTagFilterOperatorInfo(void *param) { static void sifFreeParam(SIFParam *param) { if (param == NULL) return; + taosArrayDestroy(param->result); + taosMemoryFree(param->condValue); + taosHashCleanup(param->pFilter); } static int32_t sifGetOperParamNum(EOperatorType ty) { @@ -71,15 +84,70 @@ static int32_t sifGetOperParamNum(EOperatorType ty) { } return 2; } +static int32_t sifValidateColumn(SColumnNode *cn) { + // add more check + if (cn == NULL) { + return TSDB_CODE_QRY_INVALID_INPUT; + } + if (cn->colType != COLUMN_TYPE_TAG) { + return TSDB_CODE_QRY_INVALID_INPUT; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t sifGetValueFromNode(SNode *node, char **value) { + // covert data From snode; + SValueNode *vn = (SValueNode *)node; + + char * pData = nodesGetValueFromNode(vn); + SDataType *pType = &vn->node.resType; + int32_t type = pType->type; + int32_t valLen = 0; + + if (IS_VAR_DATA_TYPE(type)) { + int32_t dataLen = varDataTLen(pData); + if (type == TSDB_DATA_TYPE_JSON) { + if (*pData == TSDB_DATA_TYPE_NULL) { + dataLen = 0; + } else if (*pData == TSDB_DATA_TYPE_NCHAR) { + dataLen = varDataTLen(pData + CHAR_BYTES); + } else if (*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) { + dataLen = LONG_BYTES; + } else if (*pData == TSDB_DATA_TYPE_BOOL) { + dataLen = CHAR_BYTES; + } + dataLen += CHAR_BYTES; + } + valLen = dataLen; + } else { + valLen = pType->bytes; + } + char *tv = taosMemoryCalloc(1, valLen + 1); + if (tv == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + memcpy(tv, pData, valLen); + *value = tv; + + return TSDB_CODE_SUCCESS; +} + static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { switch (nodeType(node)) { case QUERY_NODE_VALUE: { SValueNode *vn = (SValueNode *)node; - + SIF_ERR_RET(sifGetValueFromNode(node, ¶m->condValue)); + param->colId = -1; break; } case QUERY_NODE_COLUMN: { SColumnNode *cn = (SColumnNode *)node; + /*only support tag column*/ + SIF_ERR_RET(sifValidateColumn(cn)); + param->colId = cn->colId; + memcpy(param->dbName, cn->dbName, sizeof(cn->dbName)); + memcpy(param->colName, cn->colName, sizeof(cn->colName)); break; } @@ -89,7 +157,7 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { qError("invalid length for node:%p, length: %d", node, LIST_LENGTH(nl->pNodeList)); SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - + SIF_ERR_RET(scalarGenerateSetFromList((void **)¶m->pFilter, node, nl->dataType.type)); if (taosHashPut(ctx->pRes, &node, POINTER_BYTES, param, sizeof(*param))) { taosHashCleanup(param->pFilter); qError("taosHashPut nodeList failed, size:%d", (int32_t)sizeof(*param)); @@ -163,58 +231,63 @@ static int32_t sifExecFunction(SFunctionNode *node, SIFCtx *ctx, SIFParam *outpu qError("index-filter not support buildin function"); return TSDB_CODE_QRY_INVALID_INPUT; } - -static int32_t sifLessThanFunc(SNode *left, SNode *rigth, SIFParam *output) { - // impl later - return TSDB_CODE_SUCCESS; -} -static int32_t sifLessEqualFunc(SNode *left, SNode *rigth, SIFParam *output) { - // impl later - return TSDB_CODE_SUCCESS; -} -static int32_t sifGreaterThanFunc(SNode *left, SNode *rigth, SIFParam *output) { - // impl later - return TSDB_CODE_SUCCESS; -} -static int32_t sifGreaterEqualFunc(SNode *left, SNode *rigth, SIFParam *output) { - // impl later +static int32_t sifIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) { + SIndexMultiTermQuery *mq = indexMultiTermQueryCreate(MUST); return TSDB_CODE_SUCCESS; } -static int32_t sifEqualFunc(SNode *left, SNode *rigth, SIFParam *output) { - // impl later - return TSDB_CODE_SUCCESS; +static int32_t sifLessThanFunc(SIFParam *left, SIFParam *right, SIFParam *output) { + int id = OP_TYPE_LOWER_THAN; + return sifIndex(left, right, id, output); } -static int32_t sifNotEqualFunc(SNode *left, SNode *rigth, SIFParam *output) { - // impl later - return TSDB_CODE_SUCCESS; -} -static int32_t sifInFunc(SNode *left, SNode *rigth, SIFParam *output) { - // impl later - return TSDB_CODE_SUCCESS; -} -static int32_t sifNotInFunc(SNode *left, SNode *right, SIFParam *output) { - // impl later - return TSDB_CODE_SUCCESS; -} -static int32_t sifLikeFunc(SNode *left, SNode *right, SIFParam *output) { - // impl later - return TSDB_CODE_SUCCESS; -} -static int32_t sifNotLikeFunc(SNode *left, SNode *right, SIFParam *output) { - // impl later - return TSDB_CODE_SUCCESS; +static int32_t sifLessEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) { + int id = OP_TYPE_LOWER_EQUAL; + return sifIndex(left, right, id, output); } -static int32_t sifMatchFunc(SNode *left, SNode *rigth, SIFParam *output) { - // impl later - return TSDB_CODE_SUCCESS; +static int32_t sifGreaterThanFunc(SIFParam *left, SIFParam *right, SIFParam *output) { + int id = OP_TYPE_GREATER_THAN; + return sifIndex(left, right, id, output); } -static int32_t sifNotMatchFunc(SNode *left, SNode *rigth, SIFParam *output) { - // impl later - return TSDB_CODE_SUCCESS; +static int32_t sifGreaterEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) { + int id = OP_TYPE_GREATER_EQUAL; + return sifIndex(left, right, id, output); } -static int32_t sifDefaultFunc(SNode *left, SNode *rigth, SIFParam *output) { + +static int32_t sifEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) { + int id = OP_TYPE_EQUAL; + return sifIndex(left, right, id, output); +} +static int32_t sifNotEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) { + int id = OP_TYPE_NOT_EQUAL; + return sifIndex(left, right, id, output); +} +static int32_t sifInFunc(SIFParam *left, SIFParam *right, SIFParam *output) { + int id = OP_TYPE_IN; + return sifIndex(left, right, id, output); +} +static int32_t sifNotInFunc(SIFParam *left, SIFParam *right, SIFParam *output) { + int id = OP_TYPE_NOT_IN; + return sifIndex(left, right, id, output); +} +static int32_t sifLikeFunc(SIFParam *left, SIFParam *right, SIFParam *output) { + int id = OP_TYPE_LIKE; + return sifIndex(left, right, id, output); +} +static int32_t sifNotLikeFunc(SIFParam *left, SIFParam *right, SIFParam *output) { + int id = OP_TYPE_NOT_LIKE; + return sifIndex(left, right, id, output); +} + +static int32_t sifMatchFunc(SIFParam *left, SIFParam *right, SIFParam *output) { + int id = OP_TYPE_MATCH; + return sifIndex(left, right, id, output); +} +static int32_t sifNotMatchFunc(SIFParam *left, SIFParam *right, SIFParam *output) { + int id = OP_TYPE_NMATCH; + return sifIndex(left, right, id, output); +} +static int32_t sifDefaultFunc(SIFParam *left, SIFParam *right, SIFParam *output) { // add more except return TSDB_CODE_QRY_INVALID_INPUT; } @@ -252,17 +325,18 @@ static sif_func_t sifGetOperFn(int32_t funcId) { return sifDefaultFunc; } static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { - int32_t code = 0; - SIFParam *params = NULL; - SIF_ERR_RET(sifInitOperParams(¶ms, node, ctx)); - + int32_t code = 0; int32_t nParam = sifGetOperParamNum(node->opType); if (nParam <= 1) { SIF_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } + + SIFParam *params = NULL; + SIF_ERR_RET(sifInitOperParams(¶ms, node, ctx)); + sif_func_t operFn = sifGetOperFn(node->opType); - return operFn(node->pLeft, node->pRight, output); + return operFn(¶ms[0], nParam > 1 ? ¶ms[1] : NULL, output); _return: taosMemoryFree(params); SIF_RET(code); @@ -335,7 +409,6 @@ static EDealRes sifWalkOper(SNode *pNode, void *context) { if (ctx->code) { return DEAL_RES_ERROR; } - if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return DEAL_RES_ERROR; diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 7d52abcd1b..aeb5e01175 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -258,13 +258,13 @@ void indexOptsDestroy(SIndexOpts* opts) { * */ SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType opera) { - SIndexMultiTermQuery* p = (SIndexMultiTermQuery*)taosMemoryMalloc(sizeof(SIndexMultiTermQuery)); - if (p == NULL) { + SIndexMultiTermQuery* mtq = (SIndexMultiTermQuery*)taosMemoryMalloc(sizeof(SIndexMultiTermQuery)); + if (mtq == NULL) { return NULL; } - p->opera = opera; - p->query = taosArrayInit(4, sizeof(SIndexTermQuery)); - return p; + mtq->opera = opera; + mtq->query = taosArrayInit(4, sizeof(SIndexTermQuery)); + return mtq; } void indexMultiTermQueryDestroy(SIndexMultiTermQuery* pQuery) { for (int i = 0; i < taosArrayGetSize(pQuery->query); i++) { @@ -282,23 +282,24 @@ int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EInde SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char* colName, int32_t nColName, const char* colVal, int32_t nColVal) { - SIndexTerm* t = (SIndexTerm*)taosMemoryCalloc(1, (sizeof(SIndexTerm))); - if (t == NULL) { + SIndexTerm* tm = (SIndexTerm*)taosMemoryCalloc(1, (sizeof(SIndexTerm))); + if (tm == NULL) { return NULL; } - t->suid = suid; - t->operType = oper; - t->colType = colType; + tm->suid = suid; + tm->operType = oper; + tm->colType = colType; - t->colName = (char*)taosMemoryCalloc(1, nColName + 1); - memcpy(t->colName, colName, nColName); - t->nColName = nColName; + tm->colName = (char*)taosMemoryCalloc(1, nColName + 1); + memcpy(tm->colName, colName, nColName); + tm->nColName = nColName; - t->colVal = (char*)taosMemoryCalloc(1, nColVal + 1); - memcpy(t->colVal, colVal, nColVal); - t->nColVal = nColVal; - return t; + tm->colVal = (char*)taosMemoryCalloc(1, nColVal + 1); + memcpy(tm->colVal, colVal, nColVal); + tm->nColVal = nColVal; + + return tm; } void indexTermDestroy(SIndexTerm* p) { taosMemoryFree(p->colName); From c7ca57f5573bf949e38c797b07930f178cd9a482 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 2 May 2022 00:30:47 +0800 Subject: [PATCH 05/48] feat: rollup data submit --- include/common/taosdef.h | 10 +- include/common/tdatablock.h | 3 + include/common/tmsgdef.h | 1 + source/common/src/tdatablock.c | 155 +++++++++++++++++--- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 1 + source/dnode/vnode/inc/vnode.h | 4 +- source/dnode/vnode/src/inc/tsdb.h | 1 + source/dnode/vnode/src/inc/vnodeInt.h | 2 + source/dnode/vnode/src/tsdb/tsdbFile.c | 17 ++- source/dnode/vnode/src/tsdb/tsdbOpen.c | 24 ++- source/dnode/vnode/src/tsdb/tsdbSma.c | 38 +++-- source/dnode/vnode/src/vnd/vnodeCfg.c | 11 +- source/dnode/vnode/src/vnd/vnodeCommit.c | 23 ++- source/dnode/vnode/src/vnd/vnodeSvr.c | 20 ++- 14 files changed, 249 insertions(+), 61 deletions(-) diff --git a/include/common/taosdef.h b/include/common/taosdef.h index dd1661f871..e1f8832edf 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -79,11 +79,11 @@ typedef enum { } ETsdbSmaType; typedef enum { - TSDB_RSMA_RETENTION_0 = 0, - TSDB_RSMA_RETENTION_1 = 1, - TSDB_RSMA_RETENTION_2 = 2, - TSDB_RSMA_RETENTION_MAX = 3 -} ERSmaRetention; + TSDB_RETENTION_L0 = 0, + TSDB_RETENTION_L1 = 1, + TSDB_RETENTION_L2 = 2, + TSDB_RETENTION_MAX = 3 +} ERetentionLevel; extern char *qtypeStr[]; diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 0d5262fe29..d86586a378 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -225,6 +225,9 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData); void blockDebugShowData(const SArray* dataBlocks); +void buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema* pTSchema, int32_t vgId, + tb_uid_t uid, tb_uid_t suid); + static FORCE_INLINE int32_t blockGetEncodeSize(const SSDataBlock* pBlock) { return blockDataGetSerialMetaSize(pBlock) + blockDataGetSize(pBlock); } diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index b0439c722c..36bef5e85a 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -202,6 +202,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_CREATE_SMA, "vnode-create-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT_RSMA, "vnode-submit-rsma", SSubmitReq, SSubmitRsp) // sync integration TD_DEF_MSG_TYPE(TDMT_VND_SYNC_TIMEOUT, "vnode-sync-timeout", NULL, NULL) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index e703063dfa..e4f344b88f 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -187,17 +187,17 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c int32_t i = 0; uint8_t* start = (uint8_t*)&pColumnInfoData->nullbitmap[BitmapLen(numOfRow1)]; - int32_t overCount = BitmapLen(total) - BitmapLen(numOfRow1); - while (i < len) { // size limit of pSource->nullbitmap + int32_t overCount = BitmapLen(total) - BitmapLen(numOfRow1); + while (i < len) { // size limit of pSource->nullbitmap if (i >= 1) { - start[i - 1] |= (p[i] >> remindBits); //copy remind bits + start[i - 1] |= (p[i] >> remindBits); // copy remind bits } - if (i >= overCount) { // size limit of pColumnInfoData->nullbitmap + if (i >= overCount) { // size limit of pColumnInfoData->nullbitmap return; } - start[i] |= (p[i] << shiftBits); //copy shift bits + start[i] |= (p[i] << shiftBits); // copy shift bits i += 1; } } @@ -453,7 +453,6 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd // all fit in *stopIndex = numOfRows - 1; return TSDB_CODE_SUCCESS; - } SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int32_t rowCount) { @@ -558,7 +557,7 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) { if (IS_VAR_DATA_TYPE(pCol->info.type)) { size_t metaSize = pBlock->info.rows * sizeof(int32_t); - char* tmp = taosMemoryRealloc(pCol->varmeta.offset, metaSize); // preview calloc is too small + char* tmp = taosMemoryRealloc(pCol->varmeta.offset, metaSize); // preview calloc is too small if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -940,8 +939,9 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) { copyBackToBlock(pDataBlock, pCols); int64_t p4 = taosGetTimestampUs(); - uDebug("blockDataSort complex sort:%" PRId64 ", create:%" PRId64 ", assign:%" PRId64 ", copyback:%" PRId64 ", rows:%d\n", p1 - p0, p2 - p1, - p3 - p2, p4 - p3, rows); + uDebug("blockDataSort complex sort:%" PRId64 ", create:%" PRId64 ", assign:%" PRId64 ", copyback:%" PRId64 + ", rows:%d\n", + p1 - p0, p2 - p1, p3 - p2, p4 - p3, rows); destroyTupleIndex(index); return TSDB_CODE_SUCCESS; @@ -1178,7 +1178,7 @@ void* blockDataDestroy(SSDataBlock* pBlock) { } SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { - if(pDataBlock == NULL){ + if (pDataBlock == NULL) { return NULL; } @@ -1189,7 +1189,7 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { pBlock->info.numOfCols = numOfCols; pBlock->info.hasVarCol = pDataBlock->info.hasVarCol; - pBlock->info.rowSize = pDataBlock->info.rows; + pBlock->info.rowSize = pDataBlock->info.rows; for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData colInfo = {0}; @@ -1219,7 +1219,7 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { } size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) { - return (int32_t) ((pageSize - blockDataGetSerialMetaSize(pBlock))/ blockDataGetSerialRowSize(pBlock)); + return (int32_t)((pageSize - blockDataGetSerialMetaSize(pBlock)) / blockDataGetSerialRowSize(pBlock)); } void colDataDestroy(SColumnInfoData* pColData) { @@ -1236,14 +1236,14 @@ static void doShiftBitmap(char* nullBitmap, size_t n, size_t total) { int32_t len = BitmapLen(total); int32_t newLen = BitmapLen(total - n); - if (n%8 == 0) { - memmove(nullBitmap, nullBitmap + n/8, newLen); + if (n % 8 == 0) { + memmove(nullBitmap, nullBitmap + n / 8, newLen); } else { int32_t tail = n % 8; int32_t i = 0; - uint8_t* p = (uint8_t*) nullBitmap; - while(i < len) { + uint8_t* p = (uint8_t*)nullBitmap; + while (i < len) { uint8_t v = p[i]; p[i] = 0; @@ -1270,7 +1270,7 @@ static void colDataTrimFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_ } } -int32_t blockDataTrimFirstNRows(SSDataBlock *pBlock, size_t n) { +int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n) { if (n == 0) { return TSDB_CODE_SUCCESS; } @@ -1278,7 +1278,7 @@ int32_t blockDataTrimFirstNRows(SSDataBlock *pBlock, size_t n) { if (pBlock->info.rows <= n) { blockDataCleanup(pBlock); } else { - for(int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); colDataTrimFirstNRows(pColInfoData, n, pBlock->info.rows); } @@ -1464,3 +1464,122 @@ void blockDebugShowData(const SArray* dataBlocks) { } } +/** + * @brief TODO: Assume that the final generated result it less than 3M + * + * @param pReq + * @param pDataBlocks + * @param vgId + * @param uid set as parameter temporarily // TODO: remove this parameter, and the executor should set uid in + * SDataBlock->info.uid + * @param suid // TODO: check with Liao whether suid response is reasonable + * + * TODO: colId should be set + */ +void buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema *pTSchema, int32_t vgId, tb_uid_t uid, + tb_uid_t suid) { + int32_t sz = taosArrayGetSize(pDataBlocks); + int32_t bufSize = sizeof(SSubmitReq); + for (int32_t i = 0; i < sz; ++i) { + SDataBlockInfo* pBlkInfo = &((SSDataBlock*)taosArrayGet(pDataBlocks, i))->info; + bufSize += pBlkInfo->rows * (TD_ROW_HEAD_LEN + pBlkInfo->rowSize + BitmapLen(pBlkInfo->numOfCols)); + bufSize += sizeof(SSubmitBlk); + } + + ASSERT(bufSize < 3 * 1024 * 1024); + + *pReq = taosMemoryCalloc(1, bufSize); + void* pDataBuf = *pReq; + + int32_t msgLen = sizeof(SSubmitReq); + int32_t numOfBlks = 0; + SRowBuilder rb = {0}; + tdSRowInit(&rb, 0); // TODO: use the latest version + + for (int32_t i = 0; i < sz; ++i) { + SSDataBlock* pDataBlock = taosArrayGet(pDataBlocks, i); + int32_t colNum = pDataBlock->info.numOfCols; + int32_t rows = pDataBlock->info.rows; + int32_t rowSize = pDataBlock->info.rowSize; + int64_t groupId = pDataBlock->info.groupId; + + if(rb.nCols != colNum) { + tdSRowSetTpInfo(&rb, colNum, pTSchema->flen); + } + + SSubmitBlk* pSubmitBlk = POINTER_SHIFT(pDataBuf, msgLen); + pSubmitBlk->suid = suid; + pSubmitBlk->uid = uid; + pSubmitBlk->numOfRows = rows; + + ++numOfBlks; + + msgLen += sizeof(SSubmitBlk); + int32_t dataLen = 0; + for (int32_t j = 0; j < rows; ++j) { // iterate by row + tdSRowResetBuf(&rb, POINTER_SHIFT(pDataBuf, msgLen)); // set row buf + printf("|"); + bool isStartKey = false; + for (int32_t k = 0; k < colNum; ++k) { // iterate by column + SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); + void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); + switch (pColInfoData->info.type) { + case TSDB_DATA_TYPE_TIMESTAMP: + if (!isStartKey) { + isStartKey = true; + tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, 0, 0); + } else { + tdAppendColValToRow(&rb, 2, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, 8, k); + break; + } + break; + case TSDB_DATA_TYPE_NCHAR: { + tdAppendColValToRow(&rb, 2, TSDB_DATA_TYPE_NCHAR, TD_VTYPE_NORM, var, true, 8, k); + break; + } + case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY + tdAppendColValToRow(&rb, 2, TSDB_DATA_TYPE_VARCHAR, TD_VTYPE_NORM, var, true, 8, k); + break; + } + case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_BLOB: + case TSDB_DATA_TYPE_MEDIUMBLOB: + printf("the column type %" PRIi16 " is defined but not implemented yet\n", pColInfoData->info.type); + TASSERT(0); + break; + default: + if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) { + tdAppendColValToRow(&rb, 2, pColInfoData->info.type, TD_VTYPE_NORM, var, true, 8, k); + } else { + printf("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); + TASSERT(0); + } + break; + } + } + dataLen += TD_ROW_LEN(rb.pBuf); + } + pSubmitBlk->dataLen = dataLen; + msgLen += pSubmitBlk->dataLen; + } + + (*pReq)->length = msgLen; + + (*pReq)->header.vgId = htonl(vgId); + (*pReq)->header.contLen = htonl(msgLen); + (*pReq)->length = (*pReq)->header.contLen; + (*pReq)->numOfBlocks = htonl(numOfBlks); + SSubmitBlk* blk = (SSubmitBlk*)((*pReq) + 1); + while (numOfBlks--) { + int32_t dataLen = blk->dataLen; + blk->uid = htobe64(blk->uid); + blk->suid = htobe64(blk->suid); + blk->padding = htonl(blk->padding); + blk->sversion = htonl(blk->sversion); + blk->dataLen = htonl(blk->dataLen); + blk->schemaLen = htonl(blk->schemaLen); + blk->numOfRows = htons(blk->numOfRows); + blk = (SSubmitBlk*)(blk->data + dataLen); + } +} diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index b63a7f5b32..e3816396c9 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -284,6 +284,7 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) { dmSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA, vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_CANCEL_SMA, vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA, vmProcessWriteMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_SUBMIT_RSMA, vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_MQ_VG_CHANGE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_CONSUME, vmProcessFetchMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, vmProcessWriteMsg, DEFAULT_HANDLE); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index f4fccc2f49..62deb200c9 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -37,7 +37,7 @@ #ifdef __cplusplus extern "C" { #endif -#define TSDB_VNODE_SMA_DEBUG // TODO: evaluate to remove the macro and the relative codes + // vnode typedef struct SVnode SVnode; typedef struct STsdbCfg STsdbCfg; // todo: remove @@ -145,7 +145,7 @@ struct STsdbCfg { int32_t keep2; // TODO: save to tsdb cfg file int8_t type; // ETsdbType - SRetention retentions[TSDB_RSMA_RETENTION_MAX]; + SRetention retentions[TSDB_RETENTION_MAX]; }; typedef enum { diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 427ce8c1b6..33ffe7eda6 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -72,6 +72,7 @@ struct STsdb { char *path; SVnode *pVnode; bool repoLocked; + int8_t level; // retention level TdThreadMutex mutex; STsdbMemTable *mem; STsdbMemTable *imem; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index a9ba2e1de3..1c7f6034ee 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -124,6 +124,7 @@ int32_t tsdbUpdateTbUidList(STsdb* pTsdb, STbUidStore* pUidStore); void tsdbUidStoreDestory(STbUidStore* pStore); void* tsdbUidStoreFree(STbUidStore* pStore); int32_t tsdbTriggerRSma(STsdb* pTsdb, void* pMsg, int32_t inputType); +int32_t tsdbProcessSubmitReq(STsdb* pTsdb, int64_t version, void* pReq); typedef struct { int8_t streamType; // sma or other @@ -181,6 +182,7 @@ struct SVnode { struct STbUidStore { tb_uid_t suid; + tb_uid_t uid; // TODO: just for debugging, remove when uid provided in SSDataBlock SArray* tbUids; SHashObj* uidHash; }; diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index b1af3e0461..25566d6dad 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -27,7 +27,13 @@ static const char *TSDB_FNAME_SUFFIX[] = { "rsma", // TSDB_FILE_RSMA }; -static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, char *fname); +static const char *TSDB_DIR_NAME[] = { + "tsdb", + "rsma1", + "rsma2", +}; + +static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, const char* dname, char *fname); // static int tsdbRollBackMFile(SMFile *pMFile); static int tsdbEncodeDFInfo(void **buf, SDFInfo *pInfo); static void *tsdbDecodeDFInfo(void *buf, SDFInfo *pInfo); @@ -45,7 +51,7 @@ void tsdbInitDFile(STsdb *pRepo, SDFile *pDFile, SDiskID did, int fid, uint32_t pDFile->info.magic = TSDB_FILE_INIT_MAGIC; pDFile->info.fver = tsdbGetDFSVersion(ftype); - tsdbGetFilename(REPO_ID(pRepo), fid, ver, ftype, fname); + tsdbGetFilename(REPO_ID(pRepo), fid, ver, ftype, TSDB_DIR_NAME[pRepo->level], fname); tfsInitFile(REPO_TFS(pRepo), &(pDFile->f), did, fname); } @@ -431,14 +437,15 @@ int tsdbParseDFilename(const char *fname, int *vid, int *fid, TSDB_FILE_T *ftype return 0; } -static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, char *fname) { +static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, const char *dname, char *fname) { ASSERT(ftype != TSDB_FILE_MAX); if (ftype < TSDB_FILE_MAX) { if (ver == 0) { - snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/data/v%df%d.%s", vid, vid, fid, TSDB_FNAME_SUFFIX[ftype]); + snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/data/v%df%d.%s", vid, dname, vid, fid, + TSDB_FNAME_SUFFIX[ftype]); } else { - snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/data/v%df%d.%s-ver%" PRIu32, vid, vid, fid, + snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/data/v%df%d.%s-ver%" PRIu32, vid, dname, vid, fid, TSDB_FNAME_SUFFIX[ftype], ver); } } else { diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index da363d9bc9..e18c01dc01 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -15,21 +15,21 @@ #include "tsdb.h" -static int tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *dir); +static int tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *dir, int8_t level); int tsdbOpen(SVnode *pVnode, int8_t type) { switch (type) { case TSDB_TYPE_TSDB: - return tsdbOpenImpl(pVnode, type, &VND_TSDB(pVnode), VNODE_TSDB_DIR); + return tsdbOpenImpl(pVnode, type, &VND_TSDB(pVnode), VNODE_TSDB_DIR, TSDB_RETENTION_L0); case TSDB_TYPE_TSMA: ASSERT(0); break; case TSDB_TYPE_RSMA_L0: - return tsdbOpenImpl(pVnode, type, &VND_RSMA0(pVnode), VNODE_TSDB_DIR); + return tsdbOpenImpl(pVnode, type, &VND_RSMA0(pVnode), VNODE_TSDB_DIR, TSDB_RETENTION_L0); case TSDB_TYPE_RSMA_L1: - return tsdbOpenImpl(pVnode, type, &VND_RSMA1(pVnode), VNODE_RSMA1_DIR); + return tsdbOpenImpl(pVnode, type, &VND_RSMA1(pVnode), VNODE_RSMA1_DIR, TSDB_RETENTION_L1); case TSDB_TYPE_RSMA_L2: - return tsdbOpenImpl(pVnode, type, &VND_RSMA2(pVnode), VNODE_RSMA2_DIR); + return tsdbOpenImpl(pVnode, type, &VND_RSMA2(pVnode), VNODE_RSMA2_DIR, TSDB_RETENTION_L2); default: ASSERT(0); break; @@ -37,7 +37,17 @@ int tsdbOpen(SVnode *pVnode, int8_t type) { return 0; } -int tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *dir) { +/** + * @brief + * + * @param pVnode + * @param type + * @param ppTsdb + * @param dir + * @param level retention level + * @return int + */ +int tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *dir, int8_t level) { STsdb *pTsdb = NULL; int slen = 0; @@ -55,6 +65,7 @@ int tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *dir) { sprintf(pTsdb->path, "%s%s%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path, TD_DIRSEP, dir); pTsdb->pVnode = pVnode; + pTsdb->level = level; pTsdb->repoLocked = false; taosThreadMutexInit(&pTsdb->mutex, NULL); pTsdb->fs = tsdbNewFS(REPO_CFG(pTsdb)); @@ -79,6 +90,7 @@ _err: int tsdbClose(STsdb *pTsdb) { if (pTsdb) { + // TODO: destroy mem/imem tsdbCloseFS(pTsdb); tsdbFreeFS(pTsdb->fs); taosMemoryFree(pTsdb); diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index d4532cc3ac..0e38deb17d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -105,7 +105,7 @@ typedef struct { #define RSMA_TASK_INFO_HASH_SLOT 8 struct SRSmaInfo { - void *taskInfo[TSDB_RSMA_RETENTION_2]; // qTaskInfo_t + void *taskInfo[TSDB_RETENTION_L2]; // qTaskInfo_t }; struct SSmaStat { @@ -127,7 +127,7 @@ static FORCE_INLINE void tsdbFreeTaskHandle(qTaskInfo_t *taskHandle) { } static FORCE_INLINE void *tsdbFreeRSmaInfo(SRSmaInfo *pInfo) { - for (int32_t i = 0; i < TSDB_RSMA_RETENTION_MAX; ++i) { + for (int32_t i = 0; i < TSDB_RETENTION_MAX; ++i) { if (pInfo->taskInfo[i]) { tsdbFreeTaskHandle(pInfo->taskInfo[i]); } @@ -175,7 +175,7 @@ static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, const char *msg); static FORCE_INLINE int32_t tsdbUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid); static FORCE_INLINE int32_t tsdbUpdateTbUidListImpl(STsdb *pTsdb, tb_uid_t *suid, SArray *tbUids); static FORCE_INLINE int32_t tsdbExecuteRSmaImpl(STsdb *pTsdb, const void *pMsg, int32_t inputType, - qTaskInfo_t *taskInfo, tb_uid_t suid, int8_t retention); + qTaskInfo_t *taskInfo, STSchema *pTSchema, tb_uid_t suid, tb_uid_t uid, int8_t level); // mgmt interface static int32_t tsdbDropTSmaDataImpl(STsdb *pTsdb, int64_t indexUid); @@ -1976,6 +1976,7 @@ static int32_t tsdbFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { if (!pBlock) break; tsdbUidStorePut(pStore, msgIter.suid, NULL); + pStore->uid = msgIter.uid; // TODO: remove, just for debugging } if (terrno != TSDB_CODE_SUCCESS) return -1; @@ -1983,13 +1984,15 @@ static int32_t tsdbFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { } static FORCE_INLINE int32_t tsdbExecuteRSmaImpl(STsdb *pTsdb, const void *pMsg, int32_t inputType, - qTaskInfo_t *taskInfo, tb_uid_t suid, int8_t retention) { + qTaskInfo_t *taskInfo, STSchema *pTSchema, tb_uid_t suid, + tb_uid_t uid, int8_t level) { SArray *pResult = NULL; - tsdbDebug("vgId:%d execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, REPO_ID(pTsdb), retention, taskInfo, + tsdbDebug("vgId:%d execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, REPO_ID(pTsdb), level, taskInfo, suid); + qSetStreamInput(taskInfo, pMsg, inputType); while (1) { - SSDataBlock *output; + SSDataBlock *output = NULL; uint64_t ts; if (qExecTask(taskInfo, &output, &ts) < 0) { ASSERT(false); @@ -2010,22 +2013,28 @@ static FORCE_INLINE int32_t tsdbExecuteRSmaImpl(STsdb *pTsdb, const void *pMsg, if (taosArrayGetSize(pResult) > 0) { blockDebugShowData(pResult); + STsdb *sinkTsdb = (level == TSDB_RETENTION_L1 ? pTsdb->pVnode->pRSma1 : pTsdb->pVnode->pRSma2); + SSubmitReq *pReq = NULL; + buildSubmitReqFromDataBlock(&pReq, pResult, pTSchema, TD_VID(pTsdb->pVnode),uid, suid); + tsdbProcessSubmitReq(sinkTsdb, INT64_MAX, pReq); } else { - tsdbWarn("vgId:%d no rsma_1 data generated since %s", REPO_ID(pTsdb), tstrerror(terrno)); + tsdbWarn("vgId:%d no rsma % " PRIi8 " data generated since %s", REPO_ID(pTsdb), level, tstrerror(terrno)); } - + taosArrayDestroy(pResult); return TSDB_CODE_SUCCESS; } -int32_t tsdbExecuteRSma(STsdb *pTsdb, const void *pMsg, int32_t inputType, tb_uid_t suid) { +static int32_t tsdbExecuteRSma(STsdb *pTsdb, const void *pMsg, int32_t inputType, tb_uid_t suid, tb_uid_t uid) { SSmaEnv *pEnv = REPO_RSMA_ENV(pTsdb); if (!pEnv) { // only applicable when rsma env exists return TSDB_CODE_SUCCESS; } + ASSERT(uid != 0); // TODO: remove later + SSmaStat *pStat = SMA_ENV_STAT(pEnv); SRSmaInfo *pRSmaInfo = NULL; @@ -2037,8 +2046,11 @@ int32_t tsdbExecuteRSma(STsdb *pTsdb, const void *pMsg, int32_t inputType, tb_ui } if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { - tsdbExecuteRSmaImpl(pTsdb, pMsg, inputType, pRSmaInfo->taskInfo[0], suid, TSDB_RSMA_RETENTION_1); - tsdbExecuteRSmaImpl(pTsdb, pMsg, inputType, pRSmaInfo->taskInfo[1], suid, TSDB_RSMA_RETENTION_2); + // TODO: use the proper schema instead of 0, and cache STSchema in cache + STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, suid, 0); + tsdbExecuteRSmaImpl(pTsdb, pMsg, inputType, pRSmaInfo->taskInfo[0], pTSchema, suid, uid, TSDB_RETENTION_L1); + tsdbExecuteRSmaImpl(pTsdb, pMsg, inputType, pRSmaInfo->taskInfo[1], pTSchema, suid, uid, TSDB_RETENTION_L2); + taosMemoryFree(pTSchema); } return TSDB_CODE_SUCCESS; @@ -2056,12 +2068,12 @@ int32_t tsdbTriggerRSma(STsdb *pTsdb, void *pMsg, int32_t inputType) { tsdbFetchSubmitReqSuids(pMsg, &uidStore); if (uidStore.suid != 0) { - tsdbExecuteRSma(pTsdb, pMsg, inputType, uidStore.suid); + tsdbExecuteRSma(pTsdb, pMsg, inputType, uidStore.suid, uidStore.uid); void *pIter = taosHashIterate(uidStore.uidHash, NULL); while (pIter) { tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); - tsdbExecuteRSma(pTsdb, pMsg, inputType, *pTbSuid); + tsdbExecuteRSma(pTsdb, pMsg, inputType, *pTbSuid, 0); pIter = taosHashIterate(uidStore.uidHash, pIter); } diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index 8130a00e04..7eac0389d2 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -66,7 +66,6 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { if (tjsonAddIntegerToObject(pJson, "keep0", pCfg->tsdbCfg.keep0) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "keep1", pCfg->tsdbCfg.keep1) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "keep2", pCfg->tsdbCfg.keep2) < 0) return -1; -#ifdef TSDB_VNODE_SMA_DEBUG if (pCfg->tsdbCfg.retentions[0].freq > 0) { int32_t nRetention = 1; if (pCfg->tsdbCfg.retentions[1].freq > 0) { @@ -87,7 +86,6 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { tjsonAddItemToArray(pNodeRetentions, pNodeRetention); } } -#endif if (tjsonAddIntegerToObject(pJson, "wal.vgId", pCfg->walCfg.vgId) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod) < 0) return -1; @@ -135,11 +133,11 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { if (tjsonGetNumberValue(pJson, "keep0", pCfg->tsdbCfg.keep0) < 0) return -1; if (tjsonGetNumberValue(pJson, "keep1", pCfg->tsdbCfg.keep1) < 0) return -1; if (tjsonGetNumberValue(pJson, "keep2", pCfg->tsdbCfg.keep2) < 0) return -1; -#ifdef TSDB_VNODE_SMA_DEBUG SJson *pNodeRetentions = tjsonGetObjectItem(pJson, "retentions"); - int nRetention = tjsonGetArraySize(pNodeRetentions); - ASSERT(nRetention <= TSDB_RSMA_RETENTION_MAX); - + int32_t nRetention = tjsonGetArraySize(pNodeRetentions); + if (nRetention > TSDB_RETENTION_MAX) { + nRetention = TSDB_RETENTION_MAX; + } for (int32_t i = 0; i < nRetention; ++i) { SJson *pNodeRetention = tjsonGetArrayItem(pNodeRetentions, i); ASSERT(pNodeRetention != NULL); @@ -148,7 +146,6 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { tjsonGetNumberValue(pNodeRetention, "keep", (pCfg->tsdbCfg.retentions)[i].keep); tjsonGetNumberValue(pNodeRetention, "keepUnit", (pCfg->tsdbCfg.retentions)[i].keepUnit); } -#endif if (tjsonGetNumberValue(pJson, "wal.vgId", pCfg->walCfg.vgId) < 0) return -1; if (tjsonGetNumberValue(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod) < 0) return -1; if (tjsonGetNumberValue(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod) < 0) return -1; diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 7026673231..8bf68c5d29 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -47,9 +47,26 @@ int vnodeBegin(SVnode *pVnode) { } // begin tsdb - if (tsdbBegin(pVnode->pTsdb) < 0) { - vError("vgId:%d failed to begin tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); - return -1; + if (vnodeIsRollup(pVnode)) { + if (tsdbBegin(VND_RSMA0(pVnode)) < 0) { + vError("vgId:%d failed to begin rsma0 since %s", TD_VID(pVnode), tstrerror(terrno)); + return -1; + } + + if (tsdbBegin(VND_RSMA1(pVnode)) < 0) { + vError("vgId:%d failed to begin rsma1 since %s", TD_VID(pVnode), tstrerror(terrno)); + return -1; + } + + if (tsdbBegin(VND_RSMA2(pVnode)) < 0) { + vError("vgId:%d failed to begin rsma2 since %s", TD_VID(pVnode), tstrerror(terrno)); + return -1; + } + } else { + if (tsdbBegin(pVnode->pTsdb) < 0) { + vError("vgId:%d failed to begin tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); + return -1; + } } return 0; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index eddd295e2e..91a470b118 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -461,7 +461,7 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in SSubmitRsp rsp = {0}; pRsp->code = 0; - tsdbTriggerRSma(pVnode->pTsdb, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK); + // handle the request if (tsdbInsertData(pVnode->pTsdb, version, pSubmitReq, &rsp) < 0) { pRsp->code = terrno; @@ -470,12 +470,28 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in // pRsp->msgType = TDMT_VND_SUBMIT_RSP; // vnodeProcessSubmitReq(pVnode, ptr, pRsp); - // tsdbTriggerRSma(pVnode->pTsdb, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK); // encode the response (TODO) pRsp->pCont = rpcMallocCont(sizeof(SSubmitRsp)); memcpy(pRsp->pCont, &rsp, sizeof(rsp)); pRsp->contLen = sizeof(SSubmitRsp); + tsdbTriggerRSma(pVnode->pTsdb, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK); + return 0; } + +int32_t tsdbProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) { + if(!pReq) { + terrno = TSDB_CODE_INVALID_PTR; + return TSDB_CODE_FAILED; + } + + SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; + + if (tsdbInsertData(pTsdb, version, pSubmitReq, NULL) < 0) { + return TSDB_CODE_FAILED; + } + + return TSDB_CODE_SUCCESS; +} From 5b7117884447e96964c34f12d300b4da328c956d Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 2 May 2022 09:37:24 +0800 Subject: [PATCH 06/48] feat: rollup data storage --- include/common/tdatablock.h | 2 +- source/common/src/tdatablock.c | 8 +++++++- source/dnode/vnode/src/tsdb/tsdbFile.c | 4 ++-- source/dnode/vnode/src/tsdb/tsdbSma.c | 25 +++++++++++++++++-------- 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index d86586a378..3a98102661 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -225,7 +225,7 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData); void blockDebugShowData(const SArray* dataBlocks); -void buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema* pTSchema, int32_t vgId, +int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema* pTSchema, int32_t vgId, tb_uid_t uid, tb_uid_t suid); static FORCE_INLINE int32_t blockGetEncodeSize(const SSDataBlock* pBlock) { diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index e4f344b88f..dcea167e81 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1476,7 +1476,7 @@ void blockDebugShowData(const SArray* dataBlocks) { * * TODO: colId should be set */ -void buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema *pTSchema, int32_t vgId, tb_uid_t uid, +int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema *pTSchema, int32_t vgId, tb_uid_t uid, tb_uid_t suid) { int32_t sz = taosArrayGetSize(pDataBlocks); int32_t bufSize = sizeof(SSubmitReq); @@ -1489,6 +1489,10 @@ void buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, S ASSERT(bufSize < 3 * 1024 * 1024); *pReq = taosMemoryCalloc(1, bufSize); + if(!(*pReq)) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } void* pDataBuf = *pReq; int32_t msgLen = sizeof(SSubmitReq); @@ -1582,4 +1586,6 @@ void buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, S blk->numOfRows = htons(blk->numOfRows); blk = (SSubmitBlk*)(blk->data + dataLen); } + + return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 25566d6dad..2f40698e49 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -27,7 +27,7 @@ static const char *TSDB_FNAME_SUFFIX[] = { "rsma", // TSDB_FILE_RSMA }; -static const char *TSDB_DIR_NAME[] = { +static const char *TSDB_LEVEL_DNAME[] = { "tsdb", "rsma1", "rsma2", @@ -51,7 +51,7 @@ void tsdbInitDFile(STsdb *pRepo, SDFile *pDFile, SDiskID did, int fid, uint32_t pDFile->info.magic = TSDB_FILE_INIT_MAGIC; pDFile->info.fver = tsdbGetDFSVersion(ftype); - tsdbGetFilename(REPO_ID(pRepo), fid, ver, ftype, TSDB_DIR_NAME[pRepo->level], fname); + tsdbGetFilename(REPO_ID(pRepo), fid, ver, ftype, TSDB_LEVEL_DNAME[pRepo->level], fname); tfsInitFile(REPO_TFS(pRepo), &(pDFile->f), did, fname); } diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index 0e38deb17d..362e31ff61 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -175,7 +175,8 @@ static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, const char *msg); static FORCE_INLINE int32_t tsdbUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid); static FORCE_INLINE int32_t tsdbUpdateTbUidListImpl(STsdb *pTsdb, tb_uid_t *suid, SArray *tbUids); static FORCE_INLINE int32_t tsdbExecuteRSmaImpl(STsdb *pTsdb, const void *pMsg, int32_t inputType, - qTaskInfo_t *taskInfo, STSchema *pTSchema, tb_uid_t suid, tb_uid_t uid, int8_t level); + qTaskInfo_t *taskInfo, STSchema *pTSchema, tb_uid_t suid, tb_uid_t uid, + int8_t level); // mgmt interface static int32_t tsdbDropTSmaDataImpl(STsdb *pTsdb, int64_t indexUid); @@ -1976,7 +1977,7 @@ static int32_t tsdbFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { if (!pBlock) break; tsdbUidStorePut(pStore, msgIter.suid, NULL); - pStore->uid = msgIter.uid; // TODO: remove, just for debugging + pStore->uid = msgIter.uid; // TODO: remove, just for debugging } if (terrno != TSDB_CODE_SUCCESS) return -1; @@ -1984,8 +1985,8 @@ static int32_t tsdbFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { } static FORCE_INLINE int32_t tsdbExecuteRSmaImpl(STsdb *pTsdb, const void *pMsg, int32_t inputType, - qTaskInfo_t *taskInfo, STSchema *pTSchema, tb_uid_t suid, - tb_uid_t uid, int8_t level) { + qTaskInfo_t *taskInfo, STSchema *pTSchema, tb_uid_t suid, tb_uid_t uid, + int8_t level) { SArray *pResult = NULL; tsdbDebug("vgId:%d execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, REPO_ID(pTsdb), level, taskInfo, suid); @@ -2013,10 +2014,18 @@ static FORCE_INLINE int32_t tsdbExecuteRSmaImpl(STsdb *pTsdb, const void *pMsg, if (taosArrayGetSize(pResult) > 0) { blockDebugShowData(pResult); - STsdb *sinkTsdb = (level == TSDB_RETENTION_L1 ? pTsdb->pVnode->pRSma1 : pTsdb->pVnode->pRSma2); + STsdb *sinkTsdb = (level == TSDB_RETENTION_L1 ? pTsdb->pVnode->pRSma1 : pTsdb->pVnode->pRSma2); SSubmitReq *pReq = NULL; - buildSubmitReqFromDataBlock(&pReq, pResult, pTSchema, TD_VID(pTsdb->pVnode),uid, suid); - tsdbProcessSubmitReq(sinkTsdb, INT64_MAX, pReq); + if (buildSubmitReqFromDataBlock(&pReq, pResult, pTSchema, TD_VID(pTsdb->pVnode), uid, suid) != 0) { + taosArrayDestroy(pResult); + return TSDB_CODE_FAILED; + } + if (tsdbProcessSubmitReq(sinkTsdb, INT64_MAX, pReq) != 0) { + taosArrayDestroy(pResult); + taosMemoryFreeClear(pReq); + return TSDB_CODE_FAILED; + } + taosMemoryFreeClear(pReq); } else { tsdbWarn("vgId:%d no rsma % " PRIi8 " data generated since %s", REPO_ID(pTsdb), level, tstrerror(terrno)); } @@ -2033,7 +2042,7 @@ static int32_t tsdbExecuteRSma(STsdb *pTsdb, const void *pMsg, int32_t inputType return TSDB_CODE_SUCCESS; } - ASSERT(uid != 0); // TODO: remove later + ASSERT(uid != 0); // TODO: remove later SSmaStat *pStat = SMA_ENV_STAT(pEnv); SRSmaInfo *pRSmaInfo = NULL; From 9c02b1b2731bfb82ea960594d9ded8d09c62c81c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 2 May 2022 10:44:31 +0800 Subject: [PATCH 07/48] fix(rpc): avoid invalid write of sync api --- source/libs/transport/src/transCli.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index b4d23961b2..718be6aa64 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -949,9 +949,14 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { } if (pCtx->pSem != NULL) { - tTrace("%s cli conn %p handle resp", pTransInst->label, pConn); - memcpy((char*)pCtx->pRsp, (char*)pResp, sizeof(*pResp)); + tTrace("%s cli conn %p(sync) handle resp", pTransInst->label, pConn); + if (pCtx->pRsp == NULL) { + tTrace("%s cli conn %p(sync) failed to resp, ignore", pTransInst->label, pConn); + } else { + memcpy((char*)pCtx->pRsp, (char*)pResp, sizeof(*pResp)); + } tsem_post(pCtx->pSem); + pCtx->pRsp = NULL; } else { tTrace("%s cli conn %p handle resp", pTransInst->label, pConn); pTransInst->cfp(pTransInst->parent, pResp, pEpSet); From 47410b16d23f53574fcfac67bbee0b62a9ee7f35 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 2 May 2022 03:02:34 +0000 Subject: [PATCH 08/48] refact TDB --- source/dnode/vnode/src/meta/metaTDBImpl.c | 16 +++---- source/dnode/vnode/src/meta/metaTable.c | 12 ++--- source/dnode/vnode/src/tsdb/tsdbTDBImpl.c | 2 +- source/libs/tdb/inc/tdb.h | 4 +- source/libs/tdb/src/db/tdbBtree.c | 57 +++++++++++++++++++++++ source/libs/tdb/src/db/tdbDb.c | 9 +++- source/libs/tdb/src/inc/tdbInt.h | 2 + source/libs/tdb/test/tdbTest.cpp | 4 +- 8 files changed, 87 insertions(+), 19 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index bba707076c..cb556e8630 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -289,7 +289,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) { pVal = pBuf = buf; metaEncodeTbInfo(&pBuf, pTbCfg); vLen = POINTER_DISTANCE(pBuf, buf); - ret = tdbDbPut(pMetaDb->pTbDB, pKey, kLen, pVal, vLen, &pMetaDb->txn); + ret = tdbDbInsert(pMetaDb->pTbDB, pKey, kLen, pVal, vLen, &pMetaDb->txn); if (ret < 0) { return -1; } @@ -311,7 +311,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) { pVal = pBuf = buf; metaEncodeSchemaEx(&pBuf, &schemaWrapper); vLen = POINTER_DISTANCE(pBuf, buf); - ret = tdbDbPut(pMetaDb->pSchemaDB, pKey, kLen, pVal, vLen, &pMeta->pDB->txn); + ret = tdbDbInsert(pMetaDb->pSchemaDB, pKey, kLen, pVal, vLen, &pMeta->pDB->txn); if (ret < 0) { return -1; } @@ -325,7 +325,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) { kLen = nameLen + 1 + sizeof(uid); pVal = NULL; vLen = 0; - ret = tdbDbPut(pMetaDb->pNameIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn); + ret = tdbDbInsert(pMetaDb->pNameIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn); if (ret < 0) { return -1; } @@ -336,7 +336,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) { kLen = sizeof(uid); pVal = NULL; vLen = 0; - ret = tdbDbPut(pMetaDb->pStbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn); + ret = tdbDbInsert(pMetaDb->pStbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn); if (ret < 0) { return -1; } @@ -347,7 +347,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) { kLen = sizeof(ctbIdxKey); pVal = NULL; vLen = 0; - ret = tdbDbPut(pMetaDb->pCtbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn); + ret = tdbDbInsert(pMetaDb->pCtbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn); if (ret < 0) { return -1; } @@ -362,7 +362,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) { kLen = sizeof(uid); pVal = NULL; vLen = 0; - ret = tdbDbPut(pMetaDb->pNtbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn); + ret = tdbDbInsert(pMetaDb->pNtbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn); if (ret < 0) { return -1; } @@ -530,7 +530,7 @@ int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) { int32_t kLen = sizeof(pSmaCfg->indexUid); int32_t vLen = POINTER_DISTANCE(qBuf, pBuf); - ret = tdbDbPut(pMeta->pDB->pSmaDB, key, kLen, val, vLen, &pMetaDb->txn); + ret = tdbDbInsert(pMeta->pDB->pSmaDB, key, kLen, val, vLen, &pMetaDb->txn); if (ret < 0) { taosMemoryFreeClear(pBuf); return -1; @@ -545,7 +545,7 @@ int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) { val = NULL; vLen = 0; - ret = tdbDbPut(pMeta->pDB->pSmaIdx, key, kLen, val, vLen, &pMetaDb->txn); + ret = tdbDbInsert(pMeta->pDB->pSmaIdx, key, kLen, val, vLen, &pMetaDb->txn); if (ret < 0) { taosMemoryFreeClear(pBuf); return -1; diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index cb0b3dc81c..41ef231dcf 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -218,7 +218,7 @@ static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) { tCoderClear(&coder); // write to table.db - if (tdbDbPut(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) { + if (tdbDbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) { goto _err; } @@ -231,11 +231,11 @@ _err: } static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbDbPut(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn); + return tdbDbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn); } static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbDbPut(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); + return tdbDbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); } static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) { @@ -258,12 +258,12 @@ static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) { ttlKey.dtime = ctime + ttlDays * 24 * 60 * 60; ttlKey.uid = pME->uid; - return tdbDbPut(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, &pMeta->txn); + return tdbDbInsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, &pMeta->txn); } static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) { SCtbIdxKey ctbIdxKey = {.suid = pME->ctbEntry.suid, .uid = pME->uid}; - return tdbDbPut(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn); + return tdbDbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn); } static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pME) { @@ -304,7 +304,7 @@ static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) { tCoderInit(&coder, TD_LITTLE_ENDIAN, pVal, vLen, TD_ENCODER); tEncodeSSchemaWrapper(&coder, pSW); - if (tdbDbPut(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, &pMeta->txn) < 0) { + if (tdbDbInsert(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, &pMeta->txn) < 0) { rcode = -1; goto _exit; } diff --git a/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c b/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c index 74878e817f..8a553e94fb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c @@ -97,7 +97,7 @@ int32_t tsdbCloseDBF(SDBFile *pDBF) { int32_t tsdbSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn) { int32_t ret; - ret = tdbDbPut(pDBF->pDB, pKey, keyLen, pVal, valLen, txn); + ret = tdbDbInsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn); if (ret < 0) { tsdbError("Failed to create insert sma data into db, ret = %d", ret); return -1; diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 49da6fd8ed..dae085c672 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -40,7 +40,9 @@ int tdbCommit(TENV *pEnv, TXN *pTxn); int tdbDbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TENV *pEnv, TDB **ppDb); int tdbDbClose(TDB *pDb); int tdbDbDrop(TDB *pDb); -int tdbDbPut(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn); +int tdbDbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn); +int tdbDbDelete(TDB *pDb, const void *pKey, int kLen, TXN *pTxn); +int tdbUpsert(TDB *pTDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn); int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen); int tdbDbPGet(TDB *pDb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 2665dd1cf6..d711b47838 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -202,6 +202,36 @@ int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, in return 0; } +int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn) { + SBTC btc; + int c; + int ret; + + tdbBtcOpen(&btc, pBt, pTxn); + + // move the cursor + ret = tdbBtcMoveTo(&btc, pKey, kLen, &c); + if (ret < 0) { + tdbBtcClose(&btc); + ASSERT(0); + return -1; + } + + if (btc.idx < 0 || c != 0) { + tdbBtcClose(&btc); + return -1; + } + + // delete the key + if (tdbBtcDelete(&btc) < 0) { + tdbBtcClose(&btc); + return -1; + } + + tdbBtcClose(&btc); + return 0; +} + int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen) { return tdbBtreePGet(pBt, pKey, kLen, NULL, NULL, ppVal, vLen); } @@ -1363,6 +1393,33 @@ int tdbBtcGet(SBTC *pBtc, const void **ppKey, int *kLen, const void **ppVal, int return 0; } +int tdbBtcDelete(SBTC *pBtc) { +#if 0 + // check transaction + if (!TDB_TXN_IS_WRITE(pBtc->pTxn)) { + return -1; + } + + int idx = pBtc->idx; + int nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage); + + tdbPagerWrite(pBtc->pBt->pPager, pBtc->pPage); + + tdbPageDropCell(pBtc->pPage, idx); + + if (idx == nCells - 1) { + // drop cells above + for (;;) { + /* code */ + } + + // do balance + } +#endif + + return 0; +} + int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { int ret; int nCells; diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index aadb9238b0..f9ef2e7901 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -75,10 +75,17 @@ int tdbDbDrop(TDB *pDb) { return 0; } -int tdbDbPut(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn) { +int tdbDbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn) { return tdbBtreeInsert(pDb->pBt, pKey, keyLen, pVal, valLen, pTxn); } +int tdbDbDelete(TDB *pDb, const void *pKey, int kLen, TXN *pTxn) { return tdbBtreeDelete(pDb->pBt, pKey, kLen, pTxn); } + +int tdbUpsert(TDB *pTDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn) { + // TODO + return 0; +} + int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) { return tdbBtreeGet(pDb->pBt, pKey, kLen, ppVal, vLen); } diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index 4c19dee03e..ec0ad1fa55 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -128,6 +128,7 @@ struct SBTC { int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, tdb_cmpr_fn_t kcmpr, SBTree **ppBt); int tdbBtreeClose(SBTree *pBt); int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn); +int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn); int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen); int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); @@ -141,6 +142,7 @@ int tdbBtcMoveToNext(SBTC *pBtc); int tdbBtcMoveToPrev(SBTC *pBtc); int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen); int tdbBtcGet(SBTC *pBtc, const void **ppKey, int *kLen, const void **ppVal, int *vLen); +int tdbBtcDelete(SBTC *pBtc); // tdbPager.c ==================================== diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 1f6f28261f..9b4bc46ea9 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -152,7 +152,7 @@ TEST(tdb_test, simple_test) { for (int iData = 1; iData <= nData; iData++) { sprintf(key, "key%d", iData); sprintf(val, "value%d", iData); - ret = tdbDbPut(pDb, key, strlen(key), val, strlen(val), &txn); + ret = tdbDbInsert(pDb, key, strlen(key), val, strlen(val), &txn); GTEST_ASSERT_EQ(ret, 0); // if pool is full, commit the transaction and start a new one @@ -269,7 +269,7 @@ TEST(tdb_test, simple_test2) { for (int iData = 1; iData <= nData; iData++) { sprintf(key, "key%d", iData); sprintf(val, "value%d", iData); - ret = tdbDbPut(pDb, key, strlen(key), val, strlen(val), &txn); + ret = tdbDbInsert(pDb, key, strlen(key), val, strlen(val), &txn); GTEST_ASSERT_EQ(ret, 0); } From 52d157718b588df2f50b0db3d13c4799b4091108 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 2 May 2022 07:55:30 +0000 Subject: [PATCH 09/48] more tdb delte --- source/libs/tdb/src/db/tdbBtree.c | 75 +++++++++++++++++++++++++------ source/libs/tdb/src/db/tdbPage.c | 5 +++ source/libs/tdb/src/inc/tdbInt.h | 2 + 3 files changed, 68 insertions(+), 14 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index d711b47838..1d697e0035 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -1394,32 +1394,79 @@ int tdbBtcGet(SBTC *pBtc, const void **ppKey, int *kLen, const void **ppVal, int } int tdbBtcDelete(SBTC *pBtc) { -#if 0 - // check transaction - if (!TDB_TXN_IS_WRITE(pBtc->pTxn)) { + int idx = pBtc->idx; + int nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage); + SPager *pPager = pBtc->pBt->pPager; + const void *pKey; + i8 iPage; + SPage *pPage; + SPgno pgno; + SCell *pCell; + int szCell; + int nKey; + int ret; + + ASSERT(idx >= 0 && idx < nCells); + + // drop the cell on the leaf + ret = tdbPagerWrite(pPager, pBtc->pPage); + if (ret < 0) { + ASSERT(0); return -1; } - int idx = pBtc->idx; - int nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage); - - tdbPagerWrite(pBtc->pBt->pPager, pBtc->pPage); - tdbPageDropCell(pBtc->pPage, idx); + // update interior page or do balance if (idx == nCells - 1) { - // drop cells above - for (;;) { - /* code */ + if (idx) { + pBtc->idx--; + tdbBtcGet(pBtc, &pKey, &nKey, NULL, NULL); + + // loop to update the interial page + pgno = TDB_PAGE_PGNO(pBtc->pPage); + for (iPage = pBtc->iPage - 1; iPage >= 0; iPage--) { + pPage = pBtc->pgStack[iPage]; + idx = pBtc->idxStack[iPage]; + nCells = TDB_PAGE_TOTAL_CELLS(pPage); + + if (idx < nCells) { + ret = tdbPagerWrite(pPager, pPage); + if (ret < 0) { + ASSERT(0); + return -1; + } + + // update the cell with new key + pCell = tdbOsMalloc(nKey + 9); + tdbBtreeEncodeCell(pPage, pKey, nKey, &pgno, sizeof(pgno), pCell, &szCell); + + ret = tdbPageUpdateCell(pPage, idx, pCell, szCell); + if (ret < 0) { + tdbOsFree(pCell); + ASSERT(0); + return -1; + } + tdbOsFree(pCell); + break; + } else { + pgno = TDB_PAGE_PGNO(pPage); + } + } + } else { + // delete the leaf page and do balance (TODO) } - - // do balance } -#endif return 0; } +int tdbBtcUpsert(SBTC *pBtc) { + ASSERT(0); + // TODO + return 0; +} + int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { int ret; int nCells; diff --git a/source/libs/tdb/src/db/tdbPage.c b/source/libs/tdb/src/db/tdbPage.c index f0fdd82944..92272fb438 100644 --- a/source/libs/tdb/src/db/tdbPage.c +++ b/source/libs/tdb/src/db/tdbPage.c @@ -171,6 +171,11 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl return 0; } +int tdbPageUpdateCell(SPage *pPage, int idx, SCell *pCell, int szCell) { + tdbPageDropCell(pPage, idx); + return tdbPageInsertCell(pPage, idx, pCell, szCell, 0); +} + int tdbPageDropCell(SPage *pPage, int idx) { int lidx; SCell *pCell; diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index ec0ad1fa55..d1a1109381 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -143,6 +143,7 @@ int tdbBtcMoveToPrev(SBTC *pBtc); int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen); int tdbBtcGet(SBTC *pBtc, const void **ppKey, int *kLen, const void **ppVal, int *vLen); int tdbBtcDelete(SBTC *pBtc); +int tdbBtcUpsert(SBTC *pBtc); // tdbPager.c ==================================== @@ -280,6 +281,7 @@ void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)); int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl); int tdbPageDropCell(SPage *pPage, int idx); +int tdbPageUpdateCell(SPage *pPage, int idx, SCell *pCell, int szCell); void tdbPageCopy(SPage *pFromPage, SPage *pToPage); int tdbPageCapacity(int pageSize, int amHdrSize); From 9d0d5295f40517fd474eea94765301e8baf187e7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 2 May 2022 10:13:45 +0000 Subject: [PATCH 10/48] more TDB delte --- source/libs/tdb/src/db/tdbBtree.c | 4 +++- source/libs/tdb/src/inc/tdbInt.h | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 1d697e0035..887ab10ec0 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -1455,13 +1455,15 @@ int tdbBtcDelete(SBTC *pBtc) { } } else { // delete the leaf page and do balance (TODO) + ASSERT(TDB_PAGE_TOTAL_CELLS(pBtc->pPage) == 0); + ASSERT(0); } } return 0; } -int tdbBtcUpsert(SBTC *pBtc) { +int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pVal, int vLen) { ASSERT(0); // TODO return 0; diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index d1a1109381..e18952505c 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -143,7 +143,7 @@ int tdbBtcMoveToPrev(SBTC *pBtc); int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen); int tdbBtcGet(SBTC *pBtc, const void **ppKey, int *kLen, const void **ppVal, int *vLen); int tdbBtcDelete(SBTC *pBtc); -int tdbBtcUpsert(SBTC *pBtc); +int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pVal, int vLen); // tdbPager.c ==================================== From 4ca81bec897c5951b2d3c95cbe702259c7cdcf21 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 2 May 2022 18:16:43 +0800 Subject: [PATCH 11/48] feat: rollup data query --- source/dnode/vnode/src/inc/tsdb.h | 1 + source/dnode/vnode/src/tsdb/tsdbFS.c | 20 ++-- source/dnode/vnode/src/tsdb/tsdbFile.c | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 113 ++++++++++++++--------- source/dnode/vnode/src/vnd/vnodeCommit.c | 24 ++++- 5 files changed, 105 insertions(+), 55 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 33ffe7eda6..078a493773 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -186,6 +186,7 @@ struct STsdbFS { #define REPO_ID(r) TD_VID((r)->pVnode) #define REPO_CFG(r) (&(r)->pVnode->config.tsdbCfg) +#define REPO_LEVEL(r) ((r)->level) #define REPO_FS(r) ((r)->fs) #define REPO_META(r) ((r)->pVnode->pMeta) #define REPO_TFS(r) ((r)->pVnode->pTfs) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 4ff3227e08..6eda476b65 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -15,6 +15,8 @@ #include "tsdb.h" +extern const char *TSDB_LEVEL_DNAME[]; + typedef enum { TSDB_TXN_TEMP_FILE = 0, TSDB_TXN_CURR_FILE } TSDB_TXN_FILE_T; static const char *tsdbTxnFname[] = {"current.t", "current"}; #define TSDB_MAX_FSETS(keep, days) ((keep) / (days) + 3) @@ -35,12 +37,12 @@ static void tsdbScanAndTryFixDFilesHeader(STsdb *pRepo, int32_t *nExpired); // static int tsdbProcessExpiredFS(STsdb *pRepo); // static int tsdbCreateMeta(STsdb *pRepo); -static void tsdbGetRootDir(int repoid, char dirName[]) { - snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb", repoid); +static void tsdbGetRootDir(int repoid, int8_t level, char dirName[]) { + snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s", repoid, TSDB_LEVEL_DNAME[level]); } -static void tsdbGetDataDir(int repoid, char dirName[]) { - snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/data", repoid); +static void tsdbGetDataDir(int repoid, int8_t level, char dirName[]) { + snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/data", repoid, TSDB_LEVEL_DNAME[level]); } // For backward compatibility @@ -588,8 +590,8 @@ static int tsdbComparFidFSet(const void *arg1, const void *arg2) { } static void tsdbGetTxnFname(STsdb *pRepo, TSDB_TXN_FILE_T ftype, char fname[]) { - snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/tsdb/%s", tfsGetPrimaryPath(REPO_TFS(pRepo)), REPO_ID(pRepo), - tsdbTxnFname[ftype]); + snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/%s/%s", tfsGetPrimaryPath(REPO_TFS(pRepo)), REPO_ID(pRepo), + TSDB_LEVEL_DNAME[REPO_LEVEL(pRepo)], tsdbTxnFname[ftype]); } static int tsdbOpenFSFromCurrent(STsdb *pRepo) { @@ -719,7 +721,7 @@ static int tsdbScanRootDir(STsdb *pRepo) { STsdbFS *pfs = REPO_FS(pRepo); const STfsFile *pf; - tsdbGetRootDir(REPO_ID(pRepo), rootDir); + tsdbGetRootDir(REPO_ID(pRepo), REPO_LEVEL(pRepo), rootDir); STfsDir *tdir = tfsOpendir(REPO_TFS(pRepo), rootDir); if (tdir == NULL) { tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), rootDir, tstrerror(terrno)); @@ -753,7 +755,7 @@ static int tsdbScanDataDir(STsdb *pRepo) { STsdbFS *pfs = REPO_FS(pRepo); const STfsFile *pf; - tsdbGetDataDir(REPO_ID(pRepo), dataDir); + tsdbGetDataDir(REPO_ID(pRepo), REPO_LEVEL(pRepo), dataDir); STfsDir *tdir = tfsOpendir(REPO_TFS(pRepo), dataDir); if (tdir == NULL) { tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), dataDir, tstrerror(terrno)); @@ -801,7 +803,7 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) { regex_t regex; STsdbFS *pfs = REPO_FS(pRepo); - tsdbGetDataDir(REPO_ID(pRepo), dataDir); + tsdbGetDataDir(REPO_ID(pRepo), REPO_LEVEL(pRepo), dataDir); // Resource allocation and init regcomp(®ex, pattern, REG_EXTENDED); diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 2f40698e49..7f024786de 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -27,7 +27,7 @@ static const char *TSDB_FNAME_SUFFIX[] = { "rsma", // TSDB_FILE_RSMA }; -static const char *TSDB_LEVEL_DNAME[] = { +const char *TSDB_LEVEL_DNAME[] = { "tsdb", "rsma1", "rsma2", diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 2b8558c5ac..c597c642f7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -98,46 +98,46 @@ typedef struct SIOCostSummary { } SIOCostSummary; typedef struct SBlockLoadSuppInfo { - SColumnDataAgg *pstatis; - SColumnDataAgg **plist; - SArray *defaultLoadColumn; // default load column - int32_t *slotIds; // colId to slotId + SColumnDataAgg* pstatis; + SColumnDataAgg** plist; + SArray* defaultLoadColumn; // default load column + int32_t* slotIds; // colId to slotId } SBlockLoadSuppInfo; typedef struct STsdbReadHandle { - STsdb* pTsdb; - SQueryFilePos cur; // current position - int16_t order; - STimeWindow window; // the primary query time window that applies to all queries -// SColumnDataAgg* statis; // query level statistics, only one table block statistics info exists at any time -// SColumnDataAgg** pstatis;// the ptr array list to return to caller - int32_t numOfBlocks; - SArray* pColumns; // column list, SColumnInfoData array list - bool locateStart; - int32_t outputCapacity; - int32_t realNumOfRows; - SArray* pTableCheckInfo; // SArray - int32_t activeIndex; - bool checkFiles; // check file stage - int8_t cachelastrow; // check if last row cached - bool loadExternalRow; // load time window external data rows - bool currentLoadExternalRows; // current load external rows - int32_t loadType; // block load type - char* idStr; // query info handle, for debug purpose + STsdb* pTsdb; + SQueryFilePos cur; // current position + int16_t order; + STimeWindow window; // the primary query time window that applies to all queries + // SColumnDataAgg* statis; // query level statistics, only one table block statistics info exists at any time + // SColumnDataAgg** pstatis;// the ptr array list to return to caller + int32_t numOfBlocks; + SArray* pColumns; // column list, SColumnInfoData array list + bool locateStart; + int32_t outputCapacity; + int32_t realNumOfRows; + SArray* pTableCheckInfo; // SArray + int32_t activeIndex; + bool checkFiles; // check file stage + int8_t cachelastrow; // check if last row cached + bool loadExternalRow; // load time window external data rows + bool currentLoadExternalRows; // current load external rows + int32_t loadType; // block load type + char* idStr; // query info handle, for debug purpose int32_t type; // query type: retrieve all data blocks, 2. retrieve only last row, 3. retrieve direct prev|next rows SDFileSet* pFileGroup; SFSIter fileIter; SReadH rhelper; STableBlockInfo* pDataBlockInfo; - SDataCols* pDataCols; // in order to hold current file data block - int32_t allocSize; // allocated data block size - SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */ - SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQueryAttr */ + SDataCols* pDataCols; // in order to hold current file data block + int32_t allocSize; // allocated data block size + SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */ + SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQueryAttr */ SBlockLoadSuppInfo suppInfo; - SArray* prev; // previous row which is before than time window - SArray* next; // next row which is after the query time window - SIOCostSummary cost; - STSchema* pSchema; + SArray* prev; // previous row which is before than time window + SArray* next; // next row which is after the query time window + SIOCostSummary cost; + STSchema* pSchema; } STsdbReadHandle; typedef struct STableGroupSupporter { @@ -164,6 +164,8 @@ static int32_t tsdbCheckInfoCompar(const void* key1, const void* key2); // static void* destroyTableCheckInfo(SArray* pTableCheckInfo); static bool tsdbGetExternalRow(tsdbReaderT pHandle); +static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions); + static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) { pBlockLoadInfo->slot = -1; pBlockLoadInfo->uid = 0; @@ -350,12 +352,38 @@ static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, SQueryTableData pTsdbReadHandle->window.ekey, pTsdbReadHandle->idStr); } } - +#if 0 +int nQUERY = 0; +#endif static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions) { if (vnodeIsRollup(pVnode)) { - // for(int32_t i=0; i< TSDB_; ) { - - // } + int level = 0; +#if 1 + int64_t now = taosGetTimestamp(pVnode->config.tsdbCfg.precision); + for (int i = 0; i < TSDB_RETENTION_MAX; ++i) { + SRetention* pRetention = retentions + i; + if (pRetention->keep <= 0 || (now - pRetention->keep) >= winSKey) { + break; + } + } +#endif +#if 0 + ++nQUERY; + if(nQUERY%3 == 0) { + level = 2; + } else if(nQUERY%2 == 0) { + level = 1; + } else { + level = 0; + } +#endif + if (level == TSDB_RETENTION_L0) { + return VND_RSMA0(pVnode); + } else if (level == TSDB_RETENTION_L1) { + return VND_RSMA1(pVnode); + } else { + return VND_RSMA2(pVnode); + } } return pVnode->pTsdb; } @@ -420,8 +448,10 @@ static STsdbReadHandle* tsdbQueryTablesImpl(SVnode* pVnode, SQueryTableDataCond* } pReadHandle->suppInfo.defaultLoadColumn = getDefaultLoadColumns(pReadHandle, true); - pReadHandle->suppInfo.slotIds = taosMemoryMalloc(sizeof(int32_t) * taosArrayGetSize(pReadHandle->suppInfo.defaultLoadColumn)); - pReadHandle->suppInfo.plist = taosMemoryCalloc(taosArrayGetSize(pReadHandle->suppInfo.defaultLoadColumn), POINTER_BYTES); + pReadHandle->suppInfo.slotIds = + taosMemoryMalloc(sizeof(int32_t) * taosArrayGetSize(pReadHandle->suppInfo.defaultLoadColumn)); + pReadHandle->suppInfo.plist = + taosMemoryCalloc(taosArrayGetSize(pReadHandle->suppInfo.defaultLoadColumn), POINTER_BYTES); } pReadHandle->pDataCols = tdNewDataCols(1000, pVnode->config.tsdbCfg.maxRows); @@ -444,7 +474,6 @@ _end: tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, uint64_t taskId) { - STsdbReadHandle* pTsdbReadHandle = tsdbQueryTablesImpl(pVnode, pCond, qId, taskId); if (pTsdbReadHandle == NULL) { return NULL; @@ -462,16 +491,16 @@ tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableG return NULL; } - STableCheckInfo *pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, 0); + STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, 0); pTsdbReadHandle->pSchema = metaGetTbTSchema(pVnode->pMeta, pCheckInfo->tableId, 0); - int32_t numOfCols = taosArrayGetSize(pTsdbReadHandle->suppInfo.defaultLoadColumn); + int32_t numOfCols = taosArrayGetSize(pTsdbReadHandle->suppInfo.defaultLoadColumn); int16_t* ids = pTsdbReadHandle->suppInfo.defaultLoadColumn->pData; STSchema* pSchema = pTsdbReadHandle->pSchema; int32_t i = 0, j = 0; - while(i < numOfCols && j < pSchema->numOfCols) { + while (i < numOfCols && j < pSchema->numOfCols) { if (ids[i] == pSchema->columns[j].colId) { pTsdbReadHandle->suppInfo.slotIds[i] = j; i++; @@ -1137,7 +1166,7 @@ static int32_t doLoadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBl int32_t slotIndex) { int64_t st = taosGetTimestampUs(); - int32_t code = tdInitDataCols(pTsdbReadHandle->pDataCols, pTsdbReadHandle->pSchema); + int32_t code = tdInitDataCols(pTsdbReadHandle->pDataCols, pTsdbReadHandle->pSchema); if (code != TSDB_CODE_SUCCESS) { tsdbError("%p failed to malloc buf for pDataCols, %s", pTsdbReadHandle, pTsdbReadHandle->idStr); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 8bf68c5d29..6d8bcb35c8 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -229,10 +229,28 @@ int vnodeCommit(SVnode *pVnode) { ASSERT(0); return -1; } - if (tsdbCommit(pVnode->pTsdb) < 0) { - ASSERT(0); - return -1; + + if(vnodeIsRollup(pVnode)) { + if (tsdbCommit(VND_RSMA0(pVnode)) < 0) { + ASSERT(0); + return -1; + } + if (tsdbCommit(VND_RSMA1(pVnode)) < 0) { + ASSERT(0); + return -1; + } + if (tsdbCommit(VND_RSMA2(pVnode)) < 0) { + ASSERT(0); + return -1; + } + } else { + if (tsdbCommit(pVnode->pTsdb) < 0) { + ASSERT(0); + return -1; + } } + + if (tqCommit(pVnode->pTq) < 0) { ASSERT(0); return -1; From cca06e2b714f6faf54538f6921d9e10da7c26f58 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 2 May 2022 18:18:47 +0800 Subject: [PATCH 12/48] feat: rollup data query --- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index c597c642f7..72668a70f3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -357,7 +357,7 @@ int nQUERY = 0; #endif static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions) { if (vnodeIsRollup(pVnode)) { - int level = 0; + int level = 0; #if 1 int64_t now = taosGetTimestamp(pVnode->config.tsdbCfg.precision); for (int i = 0; i < TSDB_RETENTION_MAX; ++i) { From 26f2d342b03b09d9d2224dc02367067add49ad6c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 2 May 2022 22:15:51 +0800 Subject: [PATCH 13/48] enh(index): add prefix query --- include/libs/index/index.h | 4 +- source/libs/executor/src/indexoperator.c | 49 +++--- source/libs/index/src/index.c | 50 +----- source/libs/index/src/indexCache.c | 29 ++++ source/libs/index/src/indexTfile.c | 199 +++++++++++++++++------ 5 files changed, 220 insertions(+), 111 deletions(-) diff --git a/include/libs/index/index.h b/include/libs/index/index.h index 453b49e4c6..c33f95cb08 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -17,6 +17,7 @@ #define _TD_INDEX_H_ #include "os.h" +#include "taoserror.h" #include "tarray.h" #ifdef __cplusplus @@ -41,7 +42,8 @@ typedef enum { UPDATE_VALUE, // update index column value ADD_INDEX, // add index on specify column DROP_INDEX, // drop existed index - DROP_SATBLE // drop stable + DROP_SATBLE, // drop stable + DEFAULT // query } SIndexOperOnColumn; typedef enum { MUST = 0, SHOULD = 1, NOT = 2 } EIndexOperatorType; diff --git a/source/libs/executor/src/indexoperator.c b/source/libs/executor/src/indexoperator.c index 911ba22fe6..438c4254f7 100644 --- a/source/libs/executor/src/indexoperator.c +++ b/source/libs/executor/src/indexoperator.c @@ -55,11 +55,11 @@ typedef struct SIFParam { SArray *result; char * condValue; + uint8_t colValType; col_id_t colId; - - int64_t suid; // add later - char dbName[TSDB_DB_NAME_LEN]; - char colName[TSDB_COL_NAME_LEN]; + int64_t suid; // add later + char dbName[TSDB_DB_NAME_LEN]; + char colName[TSDB_COL_NAME_LEN]; } SIFParam; typedef int32_t (*sif_func_t)(SIFParam *left, SIFParam *rigth, SIFParam *output); @@ -145,10 +145,11 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { SColumnNode *cn = (SColumnNode *)node; /*only support tag column*/ SIF_ERR_RET(sifValidateColumn(cn)); + param->colId = cn->colId; + param->colValType = cn->node.resType.type; memcpy(param->dbName, cn->dbName, sizeof(cn->dbName)); memcpy(param->colName, cn->colName, sizeof(cn->colName)); - break; } case QUERY_NODE_NODE_LIST: { @@ -231,61 +232,68 @@ static int32_t sifExecFunction(SFunctionNode *node, SIFCtx *ctx, SIFParam *outpu qError("index-filter not support buildin function"); return TSDB_CODE_QRY_INVALID_INPUT; } -static int32_t sifIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) { - SIndexMultiTermQuery *mq = indexMultiTermQueryCreate(MUST); - return TSDB_CODE_SUCCESS; +static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) { + SIndexTerm *tm = indexTermCreate(left->suid, DEFAULT, left->colValType, left->colName, strlen(left->colName), + right->condValue, strlen(right->condValue)); + if (tm == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + SIndexMultiTermQuery *mtm = indexMultiTermQueryCreate(MUST); + indexMultiTermQueryAdd(mtm, tm, QUERY_TERM); + return indexSearch(NULL, mtm, output->result); } static int32_t sifLessThanFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_LOWER_THAN; - return sifIndex(left, right, id, output); + return sifDoIndex(left, right, id, output); } static int32_t sifLessEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_LOWER_EQUAL; - return sifIndex(left, right, id, output); + return sifDoIndex(left, right, id, output); } static int32_t sifGreaterThanFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_GREATER_THAN; - return sifIndex(left, right, id, output); + return sifDoIndex(left, right, id, output); } static int32_t sifGreaterEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_GREATER_EQUAL; - return sifIndex(left, right, id, output); + return sifDoIndex(left, right, id, output); } static int32_t sifEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_EQUAL; - return sifIndex(left, right, id, output); + return sifDoIndex(left, right, id, output); } static int32_t sifNotEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_NOT_EQUAL; - return sifIndex(left, right, id, output); + return sifDoIndex(left, right, id, output); } static int32_t sifInFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_IN; - return sifIndex(left, right, id, output); + return sifDoIndex(left, right, id, output); } static int32_t sifNotInFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_NOT_IN; - return sifIndex(left, right, id, output); + return sifDoIndex(left, right, id, output); } static int32_t sifLikeFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_LIKE; - return sifIndex(left, right, id, output); + return sifDoIndex(left, right, id, output); } static int32_t sifNotLikeFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_NOT_LIKE; - return sifIndex(left, right, id, output); + return sifDoIndex(left, right, id, output); } static int32_t sifMatchFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_MATCH; - return sifIndex(left, right, id, output); + return sifDoIndex(left, right, id, output); } static int32_t sifNotMatchFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_NMATCH; - return sifIndex(left, right, id, output); + return sifDoIndex(left, right, id, output); } static int32_t sifDefaultFunc(SIFParam *left, SIFParam *right, SIFParam *output) { // add more except @@ -460,6 +468,7 @@ static int32_t sifCalculate(SNode *pNode, SIFParam *pDst) { qError("index-filter failed to taosHashInit"); return TSDB_CODE_QRY_OUT_OF_MEMORY; } + nodesWalkExprPostOrder(pNode, sifCalcWalker, &ctx); SIF_ERR_RET(ctx.code); diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index aeb5e01175..e0c24ac3bd 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -175,55 +175,19 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { return 0; } int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result) { -#ifdef USE_LUCENE - EIndexOperatorType opera = multiQuerys->opera; - - int nQuery = taosArrayGetSize(multiQuerys->query); - char** fields = taosMemoryMalloc(sizeof(char*) * nQuery); - char** keys = taosMemoryMalloc(sizeof(char*) * nQuery); - int* types = taosMemoryMalloc(sizeof(int) * nQuery); - - for (int i = 0; i < nQuery; i++) { - SIndexTermQuery* p = taosArrayGet(multiQuerys->query, i); - SIndexTerm* term = p->field_value; - - fields[i] = taosMemoryCalloc(1, term->nKey + 1); - keys[i] = taosMemoryCalloc(1, term->nVal + 1); - - memcpy(fields[i], term->key, term->nKey); - memcpy(keys[i], term->val, term->nVal); - types[i] = (int)(p->type); - } - int* tResult = NULL; - int tsz = 0; - index_multi_search(index->index, (const char**)fields, (const char**)keys, types, nQuery, opera, &tResult, &tsz); - - for (int i = 0; i < tsz; i++) { - taosArrayPush(result, &tResult[i]); - } - - for (int i = 0; i < nQuery; i++) { - taosMemoryFree(fields[i]); - taosMemoryFree(keys[i]); - } - taosMemoryFree(fields); - taosMemoryFree(keys); - taosMemoryFree(types); -#endif - #ifdef USE_INVERTED_INDEX EIndexOperatorType opera = multiQuerys->opera; // relation of querys - SArray* interResults = taosArrayInit(4, POINTER_BYTES); + SArray* iRslts = taosArrayInit(4, POINTER_BYTES); int nQuery = taosArrayGetSize(multiQuerys->query); for (size_t i = 0; i < nQuery; i++) { - SIndexTermQuery* qTerm = taosArrayGet(multiQuerys->query, i); - SArray* tResult = NULL; - indexTermSearch(index, qTerm, &tResult); - taosArrayPush(interResults, (void*)&tResult); + SIndexTermQuery* qterm = taosArrayGet(multiQuerys->query, i); + SArray* trslt = NULL; + indexTermSearch(index, qterm, &trslt); + taosArrayPush(iRslts, (void*)&trslt); } - indexMergeFinalResults(interResults, opera, result); - indexInterResultsDestroy(interResults); + indexMergeFinalResults(iRslts, opera, result); + indexInterResultsDestroy(iRslts); #endif return 0; diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index df3c0b6e7b..eaa3c6e626 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -34,9 +34,38 @@ static char* indexCacheTermGet(const void* pData); static MemTable* indexInternalCacheCreate(int8_t type); +static int32_t cacheSearchTerm(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchPrefix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchSuffix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchRegex(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchRange(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); + +static int32_t (*cacheSearch[])(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) = { + cacheSearchTerm, cacheSearchPrefix, cacheSearchSuffix, cacheSearchRegex, cacheSearchRange}; + static void doMergeWork(SSchedMsg* msg); static bool indexCacheIteratorNext(Iterate* itera); +static int32_t cacheSearchTerm(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { + // impl later + return 0; +} +static int32_t cacheSearchPrefix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { + // impl later + return 0; +} +static int32_t cacheSearchSuffix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { + // impl later + return 0; +} +static int32_t cacheSearchRegex(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { + // impl later + return 0; +} +static int32_t cacheSearchRange(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { + // impl later + return 0; +} static IterateValue* indexCacheIteratorGetValue(Iterate* iter); IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, int8_t type) { diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 554b8092a2..7072baf574 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -57,6 +57,17 @@ static int tfileCompare(const void* a, const void* b); static int tfileParseFileName(const char* filename, uint64_t* suid, char* col, int* version); static void tfileGenFileName(char* filename, uint64_t suid, const char* col, int version); static void tfileGenFileFullName(char* fullname, const char* path, uint64_t suid, const char* col, int32_t version); +/* + * search from tfile + */ +static int32_t tfSearchTerm(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchPrefix(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchSuffix(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchRegex(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchRange(void* reader, SIndexTerm* tem, SIdxTempResult* tr); + +static int32_t (*tfSearch[])(void* reader, SIndexTerm* tem, SIdxTempResult* tr) = { + tfSearchTerm, tfSearchPrefix, tfSearchSuffix, tfSearchRegex, tfSearchRange}; TFileCache* tfileCacheCreate(const char* path) { TFileCache* tcache = taosMemoryCalloc(1, sizeof(TFileCache)); @@ -183,59 +194,153 @@ void tfileReaderDestroy(TFileReader* reader) { writerCtxDestroy(reader->ctx, reader->remove); taosMemoryFree(reader); } +static int32_t tfSearchTerm(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON); + int ret = 0; + char* p = tem->colVal; + uint64_t sz = tem->nColVal; + if (hasJson) { + p = indexPackJsonData(tem); + sz = strlen(p); + } + int64_t st = taosGetTimestampUs(); + FstSlice key = fstSliceCreate(p, sz); + uint64_t offset; + if (fstGet(((TFileReader*)reader)->fst, &key, &offset)) { + int64_t et = taosGetTimestampUs(); + int64_t cost = et - st; + indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, found table info in tindex, time cost: %" PRIu64 "us", + tem->suid, tem->colName, tem->colVal, cost); + + ret = tfileReaderLoadTableIds((TFileReader*)reader, offset, tr->total); + cost = taosGetTimestampUs() - et; + indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, load all table info, time cost: %" PRIu64 "us", tem->suid, + tem->colName, tem->colVal, cost); + } + if (hasJson) { + taosMemoryFree(p); + } + fstSliceDestroy(&key); + return 0; +} + +static int32_t tfSearchPrefix(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON); + char* p = tem->colVal; + uint64_t sz = tem->nColVal; + if (hasJson) { + p = indexPackJsonData(tem); + sz = strlen(p); + } + + SArray* offsets = taosArrayInit(16, sizeof(uint64_t)); + + AutomationCtx* ctx = automCtxCreate((void*)p, AUTOMATION_PREFIX); + FstStreamBuilder* sb = fstSearch(((TFileReader*)reader)->fst, ctx); + StreamWithState* st = streamBuilderIntoStream(sb); + StreamWithStateResult* rt = NULL; + while ((rt = streamWithStateNextWith(st, NULL)) != NULL) { + taosArrayPush(offsets, &(rt->out.out)); + swsResultDestroy(rt); + } + streamWithStateDestroy(st); + fstStreamBuilderDestroy(sb); + + int32_t ret = 0; + for (int i = 0; i < taosArrayGetSize(offsets); i++) { + uint64_t offset = *(uint64_t*)taosArrayGet(offsets, i); + ret = tfileReaderLoadTableIds((TFileReader*)reader, offset, tr->total); + if (ret != 0) { + indexError("failed to find target tablelist"); + return TSDB_CODE_TDB_FILE_CORRUPTED; + } + } + if (hasJson) { + taosMemoryFree(p); + } + return 0; +} +static int32_t tfSearchSuffix(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON); + + int ret = 0; + char* p = tem->colVal; + uint64_t sz = tem->nColVal; + if (hasJson) { + p = indexPackJsonData(tem); + sz = strlen(p); + } + int64_t st = taosGetTimestampUs(); + FstSlice key = fstSliceCreate(p, sz); + /*impl later*/ + if (hasJson) { + taosMemoryFree(p); + } + fstSliceDestroy(&key); + return 0; +} +static int32_t tfSearchRegex(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON); + + int ret = 0; + char* p = tem->colVal; + uint64_t sz = tem->nColVal; + if (hasJson) { + p = indexPackJsonData(tem); + sz = strlen(p); + } + int64_t st = taosGetTimestampUs(); + FstSlice key = fstSliceCreate(p, sz); + /*impl later*/ + + if (hasJson) { + taosMemoryFree(p); + } + fstSliceDestroy(&key); + return 0; +} +static int32_t tfSearchRange(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON); + int ret = 0; + char* p = tem->colVal; + uint64_t sz = tem->nColVal; + if (hasJson) { + p = indexPackJsonData(tem); + sz = strlen(p); + } + int64_t st = taosGetTimestampUs(); + FstSlice key = fstSliceCreate(p, sz); + // uint64_t offset; + // if (fstGet(((TFileReader*)reader)->fst, &key, &offset)) { + // int64_t et = taosGetTimestampUs(); + // int64_t cost = et - st; + // indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, found table info in tindex, time cost: %" PRIu64 "us", + // tem->suid, tem->colName, tem->colVal, cost); + + // ret = tfileReaderLoadTableIds((TFileReader*)reader, offset, tr->total); + // cost = taosGetTimestampUs() - et; + // indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, load all table info, time cost: %" PRIu64 "us", tem->suid, + // tem->colName, tem->colVal, cost); + //} + if (hasJson) { + taosMemoryFree(p); + } + fstSliceDestroy(&key); + return 0; +} int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SIdxTempResult* tr) { SIndexTerm* term = query->term; - bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON); EIndexQueryType qtype = query->qType; - - // SArray* result = taosArrayInit(16, sizeof(uint64_t)); - int ret = -1; - // refactor to callback later - if (qtype == QUERY_TERM) { - uint64_t offset; - char* p = term->colVal; - uint64_t sz = term->nColVal; - if (hasJson) { - p = indexPackJsonData(term); - sz = strlen(p); - } - int64_t st = taosGetTimestampUs(); - FstSlice key = fstSliceCreate(p, sz); - if (fstGet(reader->fst, &key, &offset)) { - int64_t et = taosGetTimestampUs(); - int64_t cost = et - st; - indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, found table info in tindex, time cost: %" PRIu64 "us", - term->suid, term->colName, term->colVal, cost); - - ret = tfileReaderLoadTableIds(reader, offset, tr->total); - cost = taosGetTimestampUs() - et; - indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, load all table info, time cost: %" PRIu64 "us", term->suid, - term->colName, term->colVal, cost); - } else { - indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, not found table info in tindex", term->suid, term->colName, - term->colVal); - } - fstSliceDestroy(&key); - if (hasJson) { - taosMemoryFree(p); - } - } else if (qtype == QUERY_PREFIX) { - // handle later - // - } else if (qtype == QUERY_SUFFIX) { - // handle later - } else if (qtype == QUERY_REGEX) { - // handle later - } else if (qtype == QUERY_RANGE) { - // handle later + if (qtype >= sizeof(tfSearch) / sizeof(tfSearch[0])) { + indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, not found table info in tindex", term->suid, term->colName, + term->colVal); + return -1; + } else { + return tfSearch[qtype](reader, term, tr); } tfileReaderUnRef(reader); - - // taosArrayAddAll(tr->total, result); - // taosArrayDestroy(result); - - return ret; + return 0; } TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int32_t version, const char* colName, uint8_t colType) { From f6e51532eda59619afb8f0dd0d4c85081d2791ed Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 2 May 2022 22:46:59 +0800 Subject: [PATCH 14/48] enh(index): add prefix query --- source/libs/index/src/indexCache.c | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index eaa3c6e626..2035766f26 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -47,7 +47,33 @@ static void doMergeWork(SSchedMsg* msg); static bool indexCacheIteratorNext(Iterate* itera); static int32_t cacheSearchTerm(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { - // impl later + if (cache == NULL) { + return 0; + } + + MemTable* mem = cache; + char* key = indexCacheTermGet(ct); + + SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC); + while (tSkipListIterNext(iter)) { + SSkipListNode* node = tSkipListIterGet(iter); + if (node == NULL) { + break; + } + CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node); + if (0 == strcmp(c->colVal, ct->colVal)) { + if (c->operaType == ADD_VALUE) { + INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid) + // taosArrayPush(result, &c->uid); + *s = kTypeValue; + } else if (c->operaType == DEL_VALUE) { + INDEX_MERGE_ADD_DEL(tr->added, tr->deled, c->uid) + } + } else { + break; + } + } + tSkipListDestroyIter(iter); return 0; } static int32_t cacheSearchPrefix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { From 05f06e04c1403a2e8889cfc8795ddb5a1507d2d2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 2 May 2022 15:17:43 +0000 Subject: [PATCH 15/48] refact sl --- include/util/tskiplist2.h | 159 +++++++++++++++++++++++++++++++ source/libs/tdb/test/tdbTest.cpp | 4 +- source/util/src/tskiplist2.c | 139 +++++++++++++++++++++++++++ 3 files changed, 301 insertions(+), 1 deletion(-) create mode 100644 include/util/tskiplist2.h create mode 100644 source/util/src/tskiplist2.c diff --git a/include/util/tskiplist2.h b/include/util/tskiplist2.h new file mode 100644 index 0000000000..ce44ff96eb --- /dev/null +++ b/include/util/tskiplist2.h @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef _TD_UTIL_SKIPLIST2_H_ +#define _TD_UTIL_SKIPLIST2_H_ + +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define SL_MAX_LEVEL 15 + +typedef struct SSkipList2 SSkipList2; +typedef struct SSLCursor SSLCursor; +typedef struct SSLCfg SSLCfg; + +typedef int32_t (*tslCmprFn)(const void *pKey1, int32_t nKey1, const void *pKey2, int32_t nKey2); + +// SSkipList2 +int32_t slOpen(const SSLCfg *pCfg, SSkipList2 **ppSl); +int32_t slClose(SSkipList2 *pSl); + +// SSLCursor +int32_t slcOpen(SSkipList2 *pSl, SSLCursor *pSlc); +int32_t slcClose(SSLCursor *pSlc); +int32_t slcMoveTo(SSLCursor *pSlc, const void *pKey, int32_t nKey); +int32_t slcMoveToNext(SSLCursor *pSlc); +int32_t slcMoveToPrev(SSLCursor *pSlc); +int32_t slcMoveToFirst(SSLCursor *pSlc); +int32_t slcMoveToLast(SSLCursor *pSlc); +int32_t slcPut(SSLCursor *pSlc, const void *pKey, int32_t nKey, const void *pData, int32_t nData); +int32_t slcGet(SSLCursor *pSlc, const void **ppKey, int32_t *nKey, const void **ppData, int32_t *nData); +int32_t slcDrop(SSLCursor *pSlc); + +// struct +struct SSLCfg { + int8_t maxLevel; + int32_t nKey; + int32_t nData; + tslCmprFn cmprFn; + void *pPool; + void *(*xMalloc)(void *, int32_t size); + void (*xFree)(void *, void *); +}; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_SKIPLIST2_H_*/ + +#if 0 +#ifndef _TD_UTIL_SKILIST_H +#define _TD_UTIL_SKILIST_H + +#include "os.h" +#include "taos.h" +#include "tarray.h" +#include "tfunctional.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define MAX_SKIP_LIST_LEVEL 15 +#define SKIP_LIST_RECORD_PERFORMANCE 0 + +// For key property setting +#define SL_ALLOW_DUP_KEY (uint8_t)0x0 // Allow duplicate key exists (for tag index usage) +#define SL_DISCARD_DUP_KEY (uint8_t)0x1 // Discard duplicate key (for data update=0 case) +#define SL_UPDATE_DUP_KEY (uint8_t)0x2 // Update duplicate key by remove/insert (for data update!=0 case) + +// For thread safety setting +#define SL_THREAD_SAFE (uint8_t)0x4 + +typedef char *SSkipListKey; +typedef char *(*__sl_key_fn_t)(const void *); + +typedef void (*sl_patch_row_fn_t)(void *pDst, const void *pSrc); +typedef void *(*iter_next_fn_t)(void *iter); + +typedef struct SSkipListNode { + uint8_t level; + void *pData; + struct SSkipListNode *forwards[]; +} SSkipListNode; + +#define SL_GET_NODE_DATA(n) (n)->pData +#define SL_NODE_GET_FORWARD_POINTER(n, l) (n)->forwards[(l)] +#define SL_NODE_GET_BACKWARD_POINTER(n, l) (n)->forwards[(n)->level + (l)] + +typedef enum { SSkipListPutSuccess = 0, SSkipListPutEarlyStop = 1, SSkipListPutSkipOne = 2 } SSkipListPutStatus; + +typedef struct SSkipList { + uint32_t seed; + __compar_fn_t comparFn; + __sl_key_fn_t keyFn; + TdThreadRwlock *lock; + uint16_t len; + uint8_t maxLevel; + uint8_t flags; + uint8_t type; // static info above + uint8_t level; + uint32_t size; + SSkipListNode *pHead; // point to the first element + SSkipListNode *pTail; // point to the last element + tGenericSavedFunc *insertHandleFn; +} SSkipList; + +typedef struct SSkipListIterator { + SSkipList *pSkipList; + SSkipListNode *cur; + int32_t step; // the number of nodes that have been checked already + int32_t order; // order of the iterator + SSkipListNode *next; // next points to the true qualified node in skiplist +} SSkipListIterator; + +#define SL_IS_THREAD_SAFE(s) (((s)->flags) & SL_THREAD_SAFE) +#define SL_DUP_MODE(s) (((s)->flags) & ((((uint8_t)1) << 2) - 1)) +#define SL_GET_NODE_KEY(s, n) ((s)->keyFn((n)->pData)) +#define SL_GET_MIN_KEY(s) SL_GET_NODE_KEY(s, SL_NODE_GET_FORWARD_POINTER((s)->pHead, 0)) +#define SL_GET_MAX_KEY(s) SL_GET_NODE_KEY((s), SL_NODE_GET_BACKWARD_POINTER((s)->pTail, 0)) +#define SL_SIZE(s) (s)->size + +SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, __compar_fn_t comparFn, uint8_t flags, + __sl_key_fn_t fn); +void tSkipListDestroy(SSkipList *pSkipList); +SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData); +void tSkipListPutBatchByIter(SSkipList *pSkipList, void *iter, iter_next_fn_t iterate); +SArray *tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey); +void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel); +SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList); +SSkipListIterator *tSkipListCreateIterFromVal(SSkipList *pSkipList, const char *val, int32_t type, int32_t order); +bool tSkipListIterNext(SSkipListIterator *iter); +SSkipListNode *tSkipListIterGet(SSkipListIterator *iter); +void *tSkipListDestroyIter(SSkipListIterator *iter); +uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key); +void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_SKILIST_H*/ + +#endif \ No newline at end of file diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 9b4bc46ea9..4f20efd904 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -316,4 +316,6 @@ TEST(tdb_test, simple_test2) { // Close Env ret = tdbEnvClose(pEnv); GTEST_ASSERT_EQ(ret, 0); -} \ No newline at end of file +} + +TEST(tdb_test, simple_delete1) { taosRemoveDir("tdb"); } \ No newline at end of file diff --git a/source/util/src/tskiplist2.c b/source/util/src/tskiplist2.c new file mode 100644 index 0000000000..6657896131 --- /dev/null +++ b/source/util/src/tskiplist2.c @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tskiplist2.h" + +typedef struct SSLNode SSLNode; +struct SSLNode { + int8_t level; + SSLNode *forwards[]; +}; + +struct SSkipList2 { + int8_t level; + uint32_t seed; + int32_t size; + const SSLCfg *pCfg; +}; + +struct SSLCursor { + SSkipList2 *pSl; + SSLNode *forwards[SL_MAX_LEVEL]; +}; + +static void *slMalloc(void *pPool, int32_t size); +static void slFree(void *pPool, void *p); +static int32_t slCmprFn(const void *pKey, int32_t nKey, const void *pData, int32_t nData); + +const SSLCfg slDefaultCfg = {.maxLevel = SL_MAX_LEVEL, + .nKey = -1, + .nData = -1, + .cmprFn = slCmprFn, + .pPool = NULL, + .xMalloc = slMalloc, + .xFree = slFree}; + +int32_t slOpen(const SSLCfg *pCfg, SSkipList2 **ppSl) { + SSkipList2 *pSl = NULL; + + *ppSl = NULL; + if (pCfg == NULL) pCfg = &slDefaultCfg; + + // check cfg (TODO) + + // create handle + pSl = pCfg->xMalloc(pCfg->pPool, sizeof(*pSl)); + // (TODO) + *ppSl = pSl; + return 0; +} + +int32_t slClose(SSkipList2 *pSl) { + // TODO + return 0; +} + +int32_t slcOpen(SSkipList2 *pSl, SSLCursor *pSlc) { + // TODO + return 0; +} + +int32_t slcClose(SSLCursor *pSlc) { + // TODO + return 0; +} + +int32_t slcMoveTo(SSLCursor *pSlc, const void *pKey, int32_t nKey) { + // TODO + return 0; +} + +int32_t slcMoveToNext(SSLCursor *pSlc) { + // TODO + return 0; +} + +int32_t slcMoveToPrev(SSLCursor *pSlc) { + // TODO + return 0; +} + +int32_t slcMoveToFirst(SSLCursor *pSlc) { + // TODO + return 0; +} + +int32_t slcMoveToLast(SSLCursor *pSlc) { + // TODO + return 0; +} + +int32_t slcPut(SSLCursor *pSlc, const void *pKey, int32_t nKey, const void *pData, int32_t nData) { + // TODO + return 0; +} + +int32_t slcGet(SSLCursor *pSlc, const void **ppKey, int32_t *nKey, const void **ppData, int32_t *nData) { + // TODO + return 0; +} + +int32_t slcDrop(SSLCursor *pSlc) { + // TODO + return 0; +} + +static FORCE_INLINE void *slMalloc(void *pPool, int32_t size) { return taosMemoryMalloc(size); } + +static FORCE_INLINE void slFree(void *pPool, void *p) { taosMemoryFree(p); } + +static int32_t slCmprFn(const void *pKey1, int32_t nKey1, const void *pKey2, int32_t nKey2) { + ASSERT(nKey1 >= 0 && nKey2 >= 0); + + int32_t nKey = nKey1 > nKey2 ? nKey2 : nKey1; + int32_t c; + + c = memcmp(pKey1, pKey2, nKey); + if (c == 0) { + if (nKey1 > nKey2) { + c = 1; + } else if (nKey1 < nKey2) { + c = -1; + } + } + + return c; +} \ No newline at end of file From 80bb3d6e544c002ab21ebc9a7a3b6d8dcabb6a57 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 3 May 2022 12:57:55 +0800 Subject: [PATCH 16/48] test: add unitest for transaction --- source/dnode/mnode/impl/inc/mndUser.h | 5 +- source/dnode/mnode/impl/src/mndUser.c | 5 +- .../mnode/impl/test/trans/CMakeLists.txt | 40 +++++-- .../impl/test/trans/{trans.cpp => trans1.cpp} | 18 +-- source/dnode/mnode/impl/test/trans/trans2.cpp | 110 ++++++++++++++++++ 5 files changed, 158 insertions(+), 20 deletions(-) rename source/dnode/mnode/impl/test/trans/{trans.cpp => trans1.cpp} (93%) create mode 100644 source/dnode/mnode/impl/test/trans/trans2.cpp diff --git a/source/dnode/mnode/impl/inc/mndUser.h b/source/dnode/mnode/impl/inc/mndUser.h index e4ecdf3283..b3eb7f2f95 100644 --- a/source/dnode/mnode/impl/inc/mndUser.h +++ b/source/dnode/mnode/impl/inc/mndUser.h @@ -24,9 +24,12 @@ extern "C" { int32_t mndInitUser(SMnode *pMnode); void mndCleanupUser(SMnode *pMnode); -SUserObj *mndAcquireUser(SMnode *pMnode, char *userName); +SUserObj *mndAcquireUser(SMnode *pMnode, const char *userName); void mndReleaseUser(SMnode *pMnode, SUserObj *pUser); +// for trans test +SSdbRaw *mndUserActionEncode(SUserObj *pUser); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 2ced813003..1800d24cf2 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -25,7 +25,6 @@ #define USER_RESERVE_SIZE 64 static int32_t mndCreateDefaultUsers(SMnode *pMnode); -static SSdbRaw *mndUserActionEncode(SUserObj *pUser); static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw); static int32_t mndUserActionInsert(SSdb *pSdb, SUserObj *pUser); static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser); @@ -90,7 +89,7 @@ static int32_t mndCreateDefaultUsers(SMnode *pMnode) { return 0; } -static SSdbRaw *mndUserActionEncode(SUserObj *pUser) { +SSdbRaw *mndUserActionEncode(SUserObj *pUser) { terrno = TSDB_CODE_OUT_OF_MEMORY; int32_t numOfReadDbs = taosHashGetSize(pUser->readDbs); @@ -238,7 +237,7 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew) { return 0; } -SUserObj *mndAcquireUser(SMnode *pMnode, char *userName) { +SUserObj *mndAcquireUser(SMnode *pMnode, const char *userName) { SSdb *pSdb = pMnode->pSdb; SUserObj *pUser = sdbAcquire(pSdb, SDB_USER, userName); if (pUser == NULL) { diff --git a/source/dnode/mnode/impl/test/trans/CMakeLists.txt b/source/dnode/mnode/impl/test/trans/CMakeLists.txt index fa0ef9f263..3931433c19 100644 --- a/source/dnode/mnode/impl/test/trans/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/trans/CMakeLists.txt @@ -1,11 +1,37 @@ -aux_source_directory(. MNODE_TRANS_TEST_SRC) -add_executable(transTest ${MNODE_TRANS_TEST_SRC}) +add_executable(transTest1 "") +target_sources(transTest1 + PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/trans1.cpp" +) target_link_libraries( - transTest + transTest1 PUBLIC sut ) - -add_test( - NAME transTest - COMMAND transTest +target_include_directories( + transTest1 + PUBLIC "${TD_SOURCE_DIR}/include/dnode/mnode" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" +) +add_test( + NAME transTest1 + COMMAND transTest1 +) + +add_executable(transTest2 "") +target_sources(transTest2 + PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/trans2.cpp" +) +target_link_libraries( + transTest2 + PUBLIC dnode mnode gtest_main +) +target_include_directories( + transTest2 + PUBLIC "${TD_SOURCE_DIR}/include/dnode/mnode" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" +) +add_test( + NAME transTest2 + COMMAND transTest2 ) diff --git a/source/dnode/mnode/impl/test/trans/trans.cpp b/source/dnode/mnode/impl/test/trans/trans1.cpp similarity index 93% rename from source/dnode/mnode/impl/test/trans/trans.cpp rename to source/dnode/mnode/impl/test/trans/trans1.cpp index bcf6fe8536..07bf9a2bcf 100644 --- a/source/dnode/mnode/impl/test/trans/trans.cpp +++ b/source/dnode/mnode/impl/test/trans/trans1.cpp @@ -11,10 +11,10 @@ #include "sut.h" -class MndTestTrans : public ::testing::Test { +class MndTestTrans1 : public ::testing::Test { protected: static void SetUpTestSuite() { - test.Init("/tmp/mnode_test_trans", 9013); + test.Init("/tmp/mnode_test_trans1", 9013); const char* fqdn = "localhost"; const char* firstEp = "localhost:9013"; server2.Start("/tmp/mnode_test_trans2", fqdn, 9020, firstEp); @@ -26,7 +26,7 @@ class MndTestTrans : public ::testing::Test { } static void KillThenRestartServer() { - char file[PATH_MAX] = "/tmp/mnode_test_trans/mnode/data/sdb.data"; + char file[PATH_MAX] = "/tmp/mnode_test_trans1/mnode/data/sdb.data"; TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); int32_t size = 3 * 1024 * 1024; void* buffer = taosMemoryMalloc(size); @@ -60,10 +60,10 @@ class MndTestTrans : public ::testing::Test { void TearDown() override {} }; -Testbase MndTestTrans::test; -TestServer MndTestTrans::server2; +Testbase MndTestTrans1::test; +TestServer MndTestTrans1::server2; -TEST_F(MndTestTrans, 00_Create_User_Crash) { +TEST_F(MndTestTrans1, 00_Create_User_Crash) { { test.SendShowReq(TSDB_MGMT_TABLE_TRANS, "trans", ""); EXPECT_EQ(test.GetShowRows(), 0); @@ -83,7 +83,7 @@ TEST_F(MndTestTrans, 00_Create_User_Crash) { } } -TEST_F(MndTestTrans, 01_Create_User_Crash) { +TEST_F(MndTestTrans1, 01_Create_User_Crash) { { SCreateUserReq createReq = {0}; strcpy(createReq.user, "u1"); @@ -107,7 +107,7 @@ TEST_F(MndTestTrans, 01_Create_User_Crash) { EXPECT_EQ(test.GetShowRows(), 2); } -TEST_F(MndTestTrans, 02_Create_Qnode1_Crash) { +TEST_F(MndTestTrans1, 02_Create_Qnode1_Crash) { { SMCreateQnodeReq createReq = {0}; createReq.dnodeId = 1; @@ -142,7 +142,7 @@ TEST_F(MndTestTrans, 02_Create_Qnode1_Crash) { } } -TEST_F(MndTestTrans, 03_Create_Qnode2_Crash) { +TEST_F(MndTestTrans1, 03_Create_Qnode2_Crash) { { SCreateDnodeReq createReq = {0}; strcpy(createReq.fqdn, "localhost"); diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp new file mode 100644 index 0000000000..ef80aa14eb --- /dev/null +++ b/source/dnode/mnode/impl/test/trans/trans2.cpp @@ -0,0 +1,110 @@ +/** + * @file trans.cpp + * @author slguan (slguan@taosdata.com) + * @brief MNODE module trans tests + * @version 1.0 + * @date 2022-05-02 + * + * @copyright Copyright (c) 2022 + * + */ + +#include + +#include "mndTrans.h" +#include "mndUser.h" +#include "tcache.h" + +void reportStartup(SMgmtWrapper *pWrapper, const char *name, const char *desc) {} + +class MndTestTrans2 : public ::testing::Test { + protected: + static void SetUpTestSuite() { + dDebugFlag = 143; + vDebugFlag = 0; + mDebugFlag = 143; + cDebugFlag = 0; + jniDebugFlag = 0; + tmrDebugFlag = 135; + uDebugFlag = 135; + rpcDebugFlag = 143; + qDebugFlag = 0; + wDebugFlag = 0; + sDebugFlag = 0; + tsdbDebugFlag = 0; + tsLogEmbedded = 1; + tsAsyncLog = 0; + + const char *logpath = "/tmp/td"; + taosRemoveDir(logpath); + taosMkDir(logpath); + tstrncpy(tsLogDir, logpath, PATH_MAX); + if (taosInitLog("taosdlog", 1) != 0) { + printf("failed to init log file\n"); + } + + walInit(); + + static SMsgCb msgCb = {0}; + msgCb.reportStartupFp = reportStartup; + msgCb.pWrapper = (SMgmtWrapper *)(&msgCb); // hack + tmsgSetDefaultMsgCb(&msgCb); + + SMnodeOpt opt = {0}; + opt.deploy = 1; + opt.replica = 1; + opt.replicas[0].id = 1; + opt.replicas[0].port = 9040; + strcpy(opt.replicas[0].fqdn, "localhost"); + opt.msgCb = msgCb; + const char *mnodepath = "/tmp/mnode_test_trans"; + pMnode = mndOpen(mnodepath, &opt); + } + + static void TearDownTestSuite() { + mndClose(pMnode); + walCleanUp(); + taosCloseLog(); + taosStopCacheRefreshWorker(); + } + + static SMnode *pMnode; + + public: + void SetUp() override {} + void TearDown() override {} + + void CreateUser(const char *user) { + SUserObj userObj = {0}; + taosEncryptPass_c((uint8_t *)"taosdata", strlen("taosdata"), userObj.pass); + tstrncpy(userObj.user, user, TSDB_USER_LEN); + tstrncpy(userObj.acct, "root", TSDB_USER_LEN); + userObj.createdTime = taosGetTimestampMs(); + userObj.updateTime = userObj.createdTime; + userObj.superUser = 1; + + SRpcMsg rpcMsg = {0}; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, &rpcMsg); + SSdbRaw *pRedoRaw = mndUserActionEncode(&userObj); + mndTransAppendRedolog(pTrans, pRedoRaw); + sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); + + char *param = strdup("====> test param <====="); + mndTransSetCb(pTrans, TEST_TRANS_START_FUNC, TEST_TRANS_STOP_FUNC, param, strlen(param) + 1); + + mndTransPrepare(pMnode, pTrans); + mndTransDrop(pTrans); + } +}; + +SMnode *MndTestTrans2::pMnode; + +TEST_F(MndTestTrans2, 01_CbFunc) { + ASSERT_NE(pMnode, nullptr); + + const char *user1 = "test1"; + CreateUser(user1); + + SUserObj *pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_NE(pUser1, nullptr); +} From f5c58a578288730547f049c9f9eb72ae3afb1b8b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 3 May 2022 08:09:58 +0000 Subject: [PATCH 17/48] more refact --- include/util/tskiplist2.h | 98 +------------------------------- source/libs/tdb/test/tdbTest.cpp | 65 ++++++++++++++++++++- 2 files changed, 65 insertions(+), 98 deletions(-) diff --git a/include/util/tskiplist2.h b/include/util/tskiplist2.h index ce44ff96eb..95fbee5a23 100644 --- a/include/util/tskiplist2.h +++ b/include/util/tskiplist2.h @@ -60,100 +60,4 @@ struct SSLCfg { } #endif -#endif /*_TD_UTIL_SKIPLIST2_H_*/ - -#if 0 -#ifndef _TD_UTIL_SKILIST_H -#define _TD_UTIL_SKILIST_H - -#include "os.h" -#include "taos.h" -#include "tarray.h" -#include "tfunctional.h" - -#ifdef __cplusplus -extern "C" { -#endif - -#define MAX_SKIP_LIST_LEVEL 15 -#define SKIP_LIST_RECORD_PERFORMANCE 0 - -// For key property setting -#define SL_ALLOW_DUP_KEY (uint8_t)0x0 // Allow duplicate key exists (for tag index usage) -#define SL_DISCARD_DUP_KEY (uint8_t)0x1 // Discard duplicate key (for data update=0 case) -#define SL_UPDATE_DUP_KEY (uint8_t)0x2 // Update duplicate key by remove/insert (for data update!=0 case) - -// For thread safety setting -#define SL_THREAD_SAFE (uint8_t)0x4 - -typedef char *SSkipListKey; -typedef char *(*__sl_key_fn_t)(const void *); - -typedef void (*sl_patch_row_fn_t)(void *pDst, const void *pSrc); -typedef void *(*iter_next_fn_t)(void *iter); - -typedef struct SSkipListNode { - uint8_t level; - void *pData; - struct SSkipListNode *forwards[]; -} SSkipListNode; - -#define SL_GET_NODE_DATA(n) (n)->pData -#define SL_NODE_GET_FORWARD_POINTER(n, l) (n)->forwards[(l)] -#define SL_NODE_GET_BACKWARD_POINTER(n, l) (n)->forwards[(n)->level + (l)] - -typedef enum { SSkipListPutSuccess = 0, SSkipListPutEarlyStop = 1, SSkipListPutSkipOne = 2 } SSkipListPutStatus; - -typedef struct SSkipList { - uint32_t seed; - __compar_fn_t comparFn; - __sl_key_fn_t keyFn; - TdThreadRwlock *lock; - uint16_t len; - uint8_t maxLevel; - uint8_t flags; - uint8_t type; // static info above - uint8_t level; - uint32_t size; - SSkipListNode *pHead; // point to the first element - SSkipListNode *pTail; // point to the last element - tGenericSavedFunc *insertHandleFn; -} SSkipList; - -typedef struct SSkipListIterator { - SSkipList *pSkipList; - SSkipListNode *cur; - int32_t step; // the number of nodes that have been checked already - int32_t order; // order of the iterator - SSkipListNode *next; // next points to the true qualified node in skiplist -} SSkipListIterator; - -#define SL_IS_THREAD_SAFE(s) (((s)->flags) & SL_THREAD_SAFE) -#define SL_DUP_MODE(s) (((s)->flags) & ((((uint8_t)1) << 2) - 1)) -#define SL_GET_NODE_KEY(s, n) ((s)->keyFn((n)->pData)) -#define SL_GET_MIN_KEY(s) SL_GET_NODE_KEY(s, SL_NODE_GET_FORWARD_POINTER((s)->pHead, 0)) -#define SL_GET_MAX_KEY(s) SL_GET_NODE_KEY((s), SL_NODE_GET_BACKWARD_POINTER((s)->pTail, 0)) -#define SL_SIZE(s) (s)->size - -SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, __compar_fn_t comparFn, uint8_t flags, - __sl_key_fn_t fn); -void tSkipListDestroy(SSkipList *pSkipList); -SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData); -void tSkipListPutBatchByIter(SSkipList *pSkipList, void *iter, iter_next_fn_t iterate); -SArray *tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey); -void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel); -SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList); -SSkipListIterator *tSkipListCreateIterFromVal(SSkipList *pSkipList, const char *val, int32_t type, int32_t order); -bool tSkipListIterNext(SSkipListIterator *iter); -SSkipListNode *tSkipListIterGet(SSkipListIterator *iter); -void *tSkipListDestroyIter(SSkipListIterator *iter); -uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key); -void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_UTIL_SKILIST_H*/ - -#endif \ No newline at end of file +#endif /*_TD_UTIL_SKIPLIST2_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 4f20efd904..78e5bc7832 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -318,4 +318,67 @@ TEST(tdb_test, simple_test2) { GTEST_ASSERT_EQ(ret, 0); } -TEST(tdb_test, simple_delete1) { taosRemoveDir("tdb"); } \ No newline at end of file +TEST(tdb_test, simple_delete1) { + int ret; + TDB *pDb; + char key[128]; + char data[128]; + TXN txn; + TENV *pEnv; + SPoolMem *pPool; + void *pKey = NULL; + void *pData = NULL; + int nKey; + int nData; + int nKV = 254; + + taosRemoveDir("tdb"); + + pPool = openPool(); + + // open env + ret = tdbEnvOpen("tdb", 4096, 256, &pEnv); + GTEST_ASSERT_EQ(ret, 0); + + // open database + ret = tdbDbOpen("db.db", -1, -1, NULL, pEnv, &pDb); + GTEST_ASSERT_EQ(ret, 0); + + tdbTxnOpen(&txn, 0, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + tdbBegin(pEnv, &txn); + + // loop to insert batch data + for (int iData = 0; iData < nKV; iData++) { + sprintf(key, "key%d", iData); + sprintf(data, "data%d", iData); + ret = tdbDbInsert(pDb, key, strlen(key), data, strlen(data), &txn); + GTEST_ASSERT_EQ(ret, 0); + } + + // query the data + for (int iData = 0; iData < nKV; iData++) { + sprintf(key, "key%d", iData); + sprintf(data, "data%d", iData); + + ret = tdbDbGet(pDb, key, strlen(key), &pData, &nData); + GTEST_ASSERT_EQ(ret, 0); + GTEST_ASSERT_EQ(memcmp(data, pData, nData), 0); + } + + // loop to delete some data + for (int iData = nKV - 1; iData >= 0; iData--) { + sprintf(key, "key%d", iData); + + ret = tdbDbDelete(pDb, key, strlen(key), &txn); + GTEST_ASSERT_EQ(ret, 0); + } + + // query the data + + tdbCommit(pEnv, &txn); + + closePool(pPool); + + tdbDbClose(pDb); + tdbEnvClose(pEnv); +} \ No newline at end of file From bf355e705a3ab4af0ff64cd821c03cf28ee686ab Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 3 May 2022 09:59:58 +0000 Subject: [PATCH 18/48] fix a TDB balance bug --- source/libs/tdb/src/db/tdbBtree.c | 8 ++++---- source/libs/tdb/test/tdbTest.cpp | 6 +++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 887ab10ec0..176a76cf28 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -582,14 +582,14 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx SCell *pCell; int szLCell, szRCell; + // balance page (iNew) and (iNew-1) for (;;) { pCell = tdbPageGetCell(pOlds[infoNews[iNew - 1].iPage], infoNews[iNew - 1].oIdx); - if (childNotLeaf) { - szLCell = szRCell = tdbBtreeCellSize(pOlds[infoNews[iNew - 1].iPage], pCell); + szLCell = tdbBtreeCellSize(pOlds[infoNews[iNew - 1].iPage], pCell); + if (!childNotLeaf) { + szRCell = szLCell; } else { - szLCell = tdbBtreeCellSize(pOlds[infoNews[iNew - 1].iPage], pCell); - int iPage = infoNews[iNew - 1].iPage; int oIdx = infoNews[iNew - 1].oIdx + 1; SPage *pPage; diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 78e5bc7832..46c13be3c2 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -202,6 +202,8 @@ TEST(tdb_test, simple_test) { ret = tdbDbcOpen(pDb, &pDBC, NULL); GTEST_ASSERT_EQ(ret, 0); + tdbDbcMoveToFirst(pDBC); + for (;;) { ret = tdbDbcNext(pDBC, &pKey, &kLen, &pVal, &vLen); if (ret < 0) break; @@ -283,6 +285,8 @@ TEST(tdb_test, simple_test2) { ret = tdbDbcOpen(pDb, &pDBC, NULL); GTEST_ASSERT_EQ(ret, 0); + tdbDbcMoveToFirst(pDBC); + for (;;) { ret = tdbDbcNext(pDBC, &pKey, &kLen, &pVal, &vLen); if (ret < 0) break; @@ -341,7 +345,7 @@ TEST(tdb_test, simple_delete1) { GTEST_ASSERT_EQ(ret, 0); // open database - ret = tdbDbOpen("db.db", -1, -1, NULL, pEnv, &pDb); + ret = tdbDbOpen("db.db", -1, -1, tKeyCmpr, pEnv, &pDb); GTEST_ASSERT_EQ(ret, 0); tdbTxnOpen(&txn, 0, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); From 6a7630e040df676e2b4850dcffaccb5abd66a637 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 3 May 2022 18:18:11 +0800 Subject: [PATCH 19/48] enh(index): add prefix query --- include/libs/index/index.h | 4 ++-- source/libs/executor/src/indexoperator.c | 21 +++++++++++++++++++-- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/include/libs/index/index.h b/include/libs/index/index.h index c33f95cb08..7b4414d329 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -46,8 +46,8 @@ typedef enum { DEFAULT // query } SIndexOperOnColumn; -typedef enum { MUST = 0, SHOULD = 1, NOT = 2 } EIndexOperatorType; -typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2, QUERY_REGEX = 3, QUERY_RANGE = 4 } EIndexQueryType; +typedef enum { MUST = 1, SHOULD, NOT } EIndexOperatorType; +typedef enum { QUERY_TERM = 1, QUERY_PREFIX, QUERY_SUFFIX, QUERY_REGEX, QUERY_RANGE } EIndexQueryType; /* * create multi query diff --git a/source/libs/executor/src/indexoperator.c b/source/libs/executor/src/indexoperator.c index 438c4254f7..87abe6a9bf 100644 --- a/source/libs/executor/src/indexoperator.c +++ b/source/libs/executor/src/indexoperator.c @@ -62,6 +62,20 @@ typedef struct SIFParam { char colName[TSDB_COL_NAME_LEN]; } SIFParam; +static int32_t sifGetFuncFromSql(EOperatorType src, EIndexQueryType *dst) { + if (src == OP_TYPE_GREATER_THAN || src == OP_TYPE_GREATER_EQUAL || src == OP_TYPE_LOWER_THAN || + src == OP_TYPE_LOWER_EQUAL) { + *dst = QUERY_RANGE; + } else if (src == OP_TYPE_EQUAL) { + *dst = QUERY_TERM; + } else if (src == OP_TYPE_LIKE || src == OP_TYPE_MATCH || src == OP_TYPE_NMATCH) { + *dst = QUERY_REGEX; + } else { + return TSDB_CODE_QRY_INVALID_INPUT; + } + return TSDB_CODE_SUCCESS; +} + typedef int32_t (*sif_func_t)(SIFParam *left, SIFParam *rigth, SIFParam *output); // construct tag filter operator later static void destroyTagFilterOperatorInfo(void *param) { @@ -238,9 +252,12 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP if (tm == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } - SIndexMultiTermQuery *mtm = indexMultiTermQueryCreate(MUST); - indexMultiTermQueryAdd(mtm, tm, QUERY_TERM); + + EIndexQueryType qtype = 0; + SIF_ERR_RET(sifGetFuncFromSql(operType, &qtype)); + + indexMultiTermQueryAdd(mtm, tm, qtype); return indexSearch(NULL, mtm, output->result); } From d3487c88c25852d63c6f99a8cf65c34692fdb407 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 3 May 2022 11:24:52 +0000 Subject: [PATCH 20/48] TDB impl delete --- source/libs/tdb/src/db/tdbBtree.c | 16 +++++++++-- source/libs/tdb/test/tdbTest.cpp | 44 ++++++++++++++++++++++++++----- 2 files changed, 51 insertions(+), 9 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 176a76cf28..85cbae316d 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -766,6 +766,13 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx } } + if (TDB_BTREE_PAGE_IS_ROOT(pParent) && TDB_PAGE_TOTAL_CELLS(pParent) == 0) { + i8 flags = TDB_BTREE_ROOT | TDB_BTREE_PAGE_IS_LEAF(pNews[0]); + // copy content to the parent page + tdbBtreeInitPage(pParent, &(SBtreeInitPageArg){.flags = flags, .pBt = pBt}, 0); + tdbPageCopy(pNews[0], pParent); + } + for (int i = 0; i < 3; i++) { if (pDivCell[i]) { tdbOsFree(pDivCell[i]); @@ -1454,9 +1461,14 @@ int tdbBtcDelete(SBTC *pBtc) { } } } else { - // delete the leaf page and do balance (TODO) + // delete the leaf page and do balance ASSERT(TDB_PAGE_TOTAL_CELLS(pBtc->pPage) == 0); - ASSERT(0); + + ret = tdbBtreeBalance(pBtc); + if (ret < 0) { + ASSERT(0); + return -1; + } } } diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 46c13be3c2..994e53cd9d 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -120,7 +120,7 @@ TEST(tdb_test, simple_test) { TENV *pEnv; TDB *pDb; tdb_cmpr_fn_t compFunc; - int nData = 10000000; + int nData = 1000000; TXN txn; taosRemoveDir("tdb"); @@ -291,9 +291,9 @@ TEST(tdb_test, simple_test2) { ret = tdbDbcNext(pDBC, &pKey, &kLen, &pVal, &vLen); if (ret < 0) break; - std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " "; - std::cout.write((char *)pVal, vLen) /* << " " << vLen */; - std::cout << std::endl; + // std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " "; + // std::cout.write((char *)pVal, vLen) /* << " " << vLen */; + // std::cout << std::endl; count++; } @@ -333,15 +333,16 @@ TEST(tdb_test, simple_delete1) { void *pKey = NULL; void *pData = NULL; int nKey; + TDBC *pDbc; int nData; - int nKV = 254; + int nKV = 69; taosRemoveDir("tdb"); pPool = openPool(); // open env - ret = tdbEnvOpen("tdb", 4096, 256, &pEnv); + ret = tdbEnvOpen("tdb", 1024, 256, &pEnv); GTEST_ASSERT_EQ(ret, 0); // open database @@ -370,7 +371,7 @@ TEST(tdb_test, simple_delete1) { } // loop to delete some data - for (int iData = nKV - 1; iData >= 0; iData--) { + for (int iData = nKV - 1; iData > 30; iData--) { sprintf(key, "key%d", iData); ret = tdbDbDelete(pDb, key, strlen(key), &txn); @@ -378,6 +379,35 @@ TEST(tdb_test, simple_delete1) { } // query the data + for (int iData = 0; iData < nKV; iData++) { + sprintf(key, "key%d", iData); + + ret = tdbDbGet(pDb, key, strlen(key), &pData, &nData); + if (iData <= 30) { + GTEST_ASSERT_EQ(ret, 0); + } else { + GTEST_ASSERT_EQ(ret, -1); + } + } + + // loop to iterate the data + tdbDbcOpen(pDb, &pDbc, NULL); + + ret = tdbDbcMoveToFirst(pDbc); + GTEST_ASSERT_EQ(ret, 0); + + pKey = NULL; + pData = NULL; + for (;;) { + ret = tdbDbcNext(pDbc, &pKey, &nKey, &pData, &nData); + if (ret < 0) break; + + std::cout.write((char *)pKey, nKey) /* << " " << kLen */ << " "; + std::cout.write((char *)pData, nData) /* << " " << vLen */; + std::cout << std::endl; + } + + tdbDbcClose(pDbc); tdbCommit(pEnv, &txn); From 8e7cf4a661d70ea1e6e1265298d6f38cd6258251 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 3 May 2022 20:14:53 +0800 Subject: [PATCH 21/48] enh(index): add prefix query --- include/libs/index/index.h | 4 ++-- source/libs/executor/src/indexoperator.c | 5 ++++- source/libs/index/src/indexCache.c | 28 +----------------------- 3 files changed, 7 insertions(+), 30 deletions(-) diff --git a/include/libs/index/index.h b/include/libs/index/index.h index 7b4414d329..5239452aa3 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -46,8 +46,8 @@ typedef enum { DEFAULT // query } SIndexOperOnColumn; -typedef enum { MUST = 1, SHOULD, NOT } EIndexOperatorType; -typedef enum { QUERY_TERM = 1, QUERY_PREFIX, QUERY_SUFFIX, QUERY_REGEX, QUERY_RANGE } EIndexQueryType; +typedef enum { MUST = 0, SHOULD, NOT } EIndexOperatorType; +typedef enum { QUERY_TERM = 0, QUERY_PREFIX, QUERY_SUFFIX, QUERY_REGEX, QUERY_RANGE } EIndexQueryType; /* * create multi query diff --git a/source/libs/executor/src/indexoperator.c b/source/libs/executor/src/indexoperator.c index 87abe6a9bf..15a3e5901a 100644 --- a/source/libs/executor/src/indexoperator.c +++ b/source/libs/executor/src/indexoperator.c @@ -258,7 +258,9 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP SIF_ERR_RET(sifGetFuncFromSql(operType, &qtype)); indexMultiTermQueryAdd(mtm, tm, qtype); - return indexSearch(NULL, mtm, output->result); + int ret = indexSearch(NULL, mtm, output->result); + indexMultiTermQueryDestroy(mtm); + return ret; } static int32_t sifLessThanFunc(SIFParam *left, SIFParam *right, SIFParam *output) { @@ -524,6 +526,7 @@ SIdxFltStatus idxGetFltStatus(SNode *pFilterNode) { if (pFilterNode == NULL) { return SFLT_NOT_INDEX; } + // impl later return SFLT_ACCURATE_INDEX; } diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 2035766f26..78368981b3 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -318,33 +318,7 @@ static int indexQueryMem(MemTable* mem, CacheTerm* ct, EIndexQueryType qtype, SI if (mem == NULL) { return 0; } - char* key = indexCacheTermGet(ct); - - SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC); - while (tSkipListIterNext(iter)) { - SSkipListNode* node = tSkipListIterGet(iter); - if (node != NULL) { - CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node); - if (qtype == QUERY_TERM) { - if (0 == strcmp(c->colVal, ct->colVal)) { - if (c->operaType == ADD_VALUE) { - INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid) - // taosArrayPush(result, &c->uid); - *s = kTypeValue; - } else if (c->operaType == DEL_VALUE) { - INDEX_MERGE_ADD_DEL(tr->added, tr->deled, c->uid) - } - } else { - break; - } - } else if (qtype == QUERY_PREFIX) { - } else if (qtype == QUERY_SUFFIX) { - } else if (qtype == QUERY_RANGE) { - } - } - } - tSkipListDestroyIter(iter); - return 0; + return cacheSearch[qtype](mem, ct, tr, s); } int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTempResult* result, STermValueType* s) { int64_t st = taosGetTimestampUs(); From dc6bdab4e4f313e0117d57b82b5a9b822ee1ea53 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 3 May 2022 21:56:05 +0800 Subject: [PATCH 22/48] enh(index): add prefix query --- include/libs/index/index.h | 4 +- source/libs/executor/src/indexoperator.c | 7 +++- source/libs/index/inc/indexInt.h | 5 ++- source/libs/index/src/index.c | 5 ++- source/libs/index/test/indexTests.cc | 50 ++++++++++++------------ source/libs/index/test/jsonUT.cc | 16 ++++---- 6 files changed, 46 insertions(+), 41 deletions(-) diff --git a/include/libs/index/index.h b/include/libs/index/index.h index 5239452aa3..bf260d4899 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -168,8 +168,8 @@ void indexOptsDestroy(SIndexOpts* opts); * @param: */ -SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn operType, uint8_t colType, const char* colName, - int32_t nColName, const char* colVal, int32_t nColVal); +SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn operType, int8_t qType, uint8_t colType, + const char* colName, int32_t nColName, const char* colVal, int32_t nColVal); void indexTermDestroy(SIndexTerm* p); /* diff --git a/source/libs/executor/src/indexoperator.c b/source/libs/executor/src/indexoperator.c index 15a3e5901a..0f075c5de2 100644 --- a/source/libs/executor/src/indexoperator.c +++ b/source/libs/executor/src/indexoperator.c @@ -247,8 +247,11 @@ static int32_t sifExecFunction(SFunctionNode *node, SIFCtx *ctx, SIFParam *outpu return TSDB_CODE_QRY_INVALID_INPUT; } static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) { - SIndexTerm *tm = indexTermCreate(left->suid, DEFAULT, left->colValType, left->colName, strlen(left->colName), - right->condValue, strlen(right->condValue)); + SIndexTerm *tm = indexTermCreate(left->suid, DEFAULT, operType, left->colValType, left->colName, + strlen(left->colName), right->condValue, strlen(right->condValue)); + if (operType == OP_TYPE_LOWER_EQUAL || operType == OP_TYPE_GREATER_EQUAL || operType == OP_TYPE_GREATER_THAN || + operType == OP_TYPE_LOWER_THAN) { + } if (tm == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 37318767c7..62ddf85985 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -57,7 +57,7 @@ struct SIndex { char* path; - SIndexStat stat; + SIndexStat stat; TdThreadMutex mtx; }; @@ -86,6 +86,7 @@ typedef struct SIndexTerm { int32_t nColName; char* colVal; int32_t nColVal; + int8_t qType; // just use for range } SIndexTerm; typedef struct SIndexTermQuery { @@ -165,7 +166,7 @@ int32_t indexSerialCacheKey(ICacheKey* key, char* buf); } while (0) #define INDEX_TYPE_CONTAIN_EXTERN_TYPE(ty, exTy) (((ty >> 4) & (exTy)) != 0) -#define INDEX_TYPE_GET_TYPE(ty) (ty & 0x0F) +#define INDEX_TYPE_GET_TYPE(ty) (ty & 0x0F) #define INDEX_TYPE_ADD_EXTERN_TYPE(ty, exTy) \ do { \ uint8_t oldTy = ty; \ diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index e0c24ac3bd..ee77aa92e9 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -244,8 +244,8 @@ int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EInde return 0; } -SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char* colName, - int32_t nColName, const char* colVal, int32_t nColVal) { +SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, int8_t queryType, uint8_t colType, + const char* colName, int32_t nColName, const char* colVal, int32_t nColVal) { SIndexTerm* tm = (SIndexTerm*)taosMemoryCalloc(1, (sizeof(SIndexTerm))); if (tm == NULL) { return NULL; @@ -262,6 +262,7 @@ SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colTy tm->colVal = (char*)taosMemoryCalloc(1, nColVal + 1); memcpy(tm->colVal, colVal, nColVal); tm->nColVal = nColVal; + tm->qType = queryType; return tm; } diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 0162b754ee..d8ea6a8233 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -483,9 +483,9 @@ TEST_F(IndexTFileEnv, test_tfile_write) { std::string colName("voltage"); std::string colVal("ab"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); - SIndexTermQuery query = { term, QUERY_TERM}; + SIndexTermQuery query = {term, QUERY_TERM}; SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t)); fObj->Get(&query, result); @@ -557,7 +557,7 @@ TEST_F(IndexCacheEnv, cache_test) { std::string colName("voltage"); { std::string colVal("v1"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); @@ -565,28 +565,28 @@ TEST_F(IndexCacheEnv, cache_test) { } { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); } { std::string colVal("v2"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); } { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); } { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); @@ -595,14 +595,14 @@ TEST_F(IndexCacheEnv, cache_test) { std::cout << "--------first----------" << std::endl; { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, othColId, version++, suid++); indexTermDestroy(term); } { std::string colVal("v4"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, othColId, version++, suid++); indexTermDestroy(term); @@ -613,7 +613,7 @@ TEST_F(IndexCacheEnv, cache_test) { std::string colVal("v4"); for (size_t i = 0; i < 10; i++) { colVal[colVal.size() - 1] = 'a' + i; - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); @@ -623,9 +623,9 @@ TEST_F(IndexCacheEnv, cache_test) { // begin query { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); - SIndexTermQuery query = { term, QUERY_TERM }; + SIndexTermQuery query = {term, QUERY_TERM}; SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid)); STermValueType valType; @@ -638,9 +638,9 @@ TEST_F(IndexCacheEnv, cache_test) { } { std::string colVal("v2"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); - SIndexTermQuery query = { term, QUERY_TERM }; + SIndexTermQuery query = {term, QUERY_TERM}; SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid)); STermValueType valType; @@ -670,7 +670,7 @@ class IndexObj { return ret; } void Del(const std::string& colName, const std::string& colVal, uint64_t uid) { - SIndexTerm* term = indexTermCreate(0, DEL_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, DEL_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -679,7 +679,7 @@ class IndexObj { } int WriteMillonData(const std::string& colName, const std::string& colVal = "Hello world", size_t numOfTable = 100 * 10000) { - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -701,7 +701,7 @@ class IndexObj { // opt tColVal[taosRand() % colValSize] = 'a' + k % 26; } - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), tColVal.c_str(), tColVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -737,7 +737,7 @@ class IndexObj { int SearchOne(const std::string& colName, const std::string& colVal) { SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); indexMultiTermQueryAdd(mq, term, QUERY_TERM); @@ -759,7 +759,7 @@ class IndexObj { } int SearchOneTarget(const std::string& colName, const std::string& colVal, uint64_t val) { SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); indexMultiTermQueryAdd(mq, term, QUERY_TERM); @@ -784,7 +784,7 @@ class IndexObj { void PutOne(const std::string& colName, const std::string& colVal) { SIndexMultiTerm* terms = indexMultiTermCreate(); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); indexMultiTermAdd(terms, term); Put(terms, 10); @@ -792,7 +792,7 @@ class IndexObj { } void PutOneTarge(const std::string& colName, const std::string& colVal, uint64_t val) { SIndexMultiTerm* terms = indexMultiTermCreate(); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); indexMultiTermAdd(terms, term); Put(terms, val); @@ -832,7 +832,7 @@ TEST_F(IndexEnv2, testIndexOpen) { { std::string colName("tag1"), colVal("Hello"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -847,7 +847,7 @@ TEST_F(IndexEnv2, testIndexOpen) { size_t size = 200; std::string colName("tag1"), colVal("hello"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -862,7 +862,7 @@ TEST_F(IndexEnv2, testIndexOpen) { size_t size = 200; std::string colName("tag1"), colVal("Hello"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -877,7 +877,7 @@ TEST_F(IndexEnv2, testIndexOpen) { { std::string colName("tag1"), colVal("Hello"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); indexMultiTermQueryAdd(mq, term, QUERY_TERM); diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index f789d23136..e1e5004701 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -40,7 +40,7 @@ TEST_F(JsonEnv, testWrite) { { std::string colName("test"); std::string colVal("ab"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); @@ -53,7 +53,7 @@ TEST_F(JsonEnv, testWrite) { { std::string colName("voltage"); std::string colVal("ab1"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); @@ -66,7 +66,7 @@ TEST_F(JsonEnv, testWrite) { { std::string colName("voltage"); std::string colVal("123"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); @@ -81,7 +81,7 @@ TEST_F(JsonEnv, testWrite) { std::string colVal("ab"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* q = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SArray* result = taosArrayInit(1, sizeof(uint64_t)); @@ -95,7 +95,7 @@ TEST_F(JsonEnv, testWriteMillonData) { { std::string colName("test"); std::string colVal("ab"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); @@ -110,7 +110,7 @@ TEST_F(JsonEnv, testWriteMillonData) { std::string colVal("abxxxxxxxxxxxx"); for (int i = 0; i < 1000; i++) { colVal[i % colVal.size()] = '0' + i % 128; - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); @@ -124,7 +124,7 @@ TEST_F(JsonEnv, testWriteMillonData) { { std::string colName("voltagefdadfa"); std::string colVal("abxxxxxxxxxxxx"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); @@ -139,7 +139,7 @@ TEST_F(JsonEnv, testWriteMillonData) { std::string colVal("ab"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* q = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SArray* result = taosArrayInit(1, sizeof(uint64_t)); From cfada1cc8ae2ee8e03baf6cf0b99ffa384336578 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 3 May 2022 14:08:30 +0000 Subject: [PATCH 23/48] more sl refact --- include/util/tskiplist2.h | 6 ++++ source/util/src/tskiplist2.c | 57 +++++++++++++++++++++++++++++------- 2 files changed, 53 insertions(+), 10 deletions(-) diff --git a/include/util/tskiplist2.h b/include/util/tskiplist2.h index 95fbee5a23..f3366dd574 100644 --- a/include/util/tskiplist2.h +++ b/include/util/tskiplist2.h @@ -32,6 +32,7 @@ typedef int32_t (*tslCmprFn)(const void *pKey1, int32_t nKey1, const void *pKey2 // SSkipList2 int32_t slOpen(const SSLCfg *pCfg, SSkipList2 **ppSl); int32_t slClose(SSkipList2 *pSl); +int32_t slClear(SSkipList2 *pSl); // SSLCursor int32_t slcOpen(SSkipList2 *pSl, SSLCursor *pSlc); @@ -56,6 +57,11 @@ struct SSLCfg { void (*xFree)(void *, void *); }; +struct SSLCursor { + SSkipList2 *pSl; + SSLNode **forwards[SL_MAX_LEVEL]; +}; + #ifdef __cplusplus } #endif diff --git a/source/util/src/tskiplist2.c b/source/util/src/tskiplist2.c index 6657896131..e7391da285 100644 --- a/source/util/src/tskiplist2.c +++ b/source/util/src/tskiplist2.c @@ -27,11 +27,7 @@ struct SSkipList2 { uint32_t seed; int32_t size; const SSLCfg *pCfg; -}; - -struct SSLCursor { - SSkipList2 *pSl; - SSLNode *forwards[SL_MAX_LEVEL]; + SSLNode *pHead[]; }; static void *slMalloc(void *pPool, int32_t size); @@ -48,25 +44,66 @@ const SSLCfg slDefaultCfg = {.maxLevel = SL_MAX_LEVEL, int32_t slOpen(const SSLCfg *pCfg, SSkipList2 **ppSl) { SSkipList2 *pSl = NULL; + int32_t size; *ppSl = NULL; if (pCfg == NULL) pCfg = &slDefaultCfg; - // check cfg (TODO) + // check config (TODO) + + // malloc handle + size = sizeof(*pSl) + sizeof(SSLNode *) * pCfg->maxLevel * 2; + pSl = pCfg->xMalloc(pCfg->pPool, size); + if (pSl == NULL) { + return -1; + } + + pSl->level = 0; + pSl->seed = taosRand(); + pSl->size = 0; + pSl->pCfg = pCfg; + + // init an empty skiplist + for (int32_t i = 0; i < pCfg->maxLevel * 2; i++) { + pSl->pHead[i] = NULL; + } - // create handle - pSl = pCfg->xMalloc(pCfg->pPool, sizeof(*pSl)); - // (TODO) *ppSl = pSl; return 0; } int32_t slClose(SSkipList2 *pSl) { - // TODO + if (pSl) { + slClear(pSl); + if (pSl->pCfg->xFree) { + pSl->pCfg->xFree(pSl->pCfg->pPool, pSl); + } + } + + return 0; +} + +int32_t slClear(SSkipList2 *pSl) { + // loop to clear sl + for (;;) { + // (TODO) + } + + // init sl (TODO) + return 0; } int32_t slcOpen(SSkipList2 *pSl, SSLCursor *pSlc) { + pSlc->pSl = pSl; + + for (int i = 0; i < SL_MAX_LEVEL; i++) { + if (i < pSl->pCfg->maxLevel) { + } else { + pSlc->forwards[i] = NULL; + } + } + // TODO return 0; } From ac0fb0237ec3ff4f0fc882322c617599e9655187 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 4 May 2022 03:57:16 +0000 Subject: [PATCH 24/48] drop table code --- include/util/tskiplist2.h | 1 + source/dnode/vnode/src/inc/meta.h | 2 - source/dnode/vnode/src/inc/vnodeInt.h | 18 +++--- source/dnode/vnode/src/meta/metaTable.c | 77 ++++++++++++++++++++++--- source/dnode/vnode/src/vnd/vnodeSvr.c | 39 +++++++++++-- source/libs/tdb/inc/tdb.h | 2 +- source/libs/tdb/src/db/tdbBtree.c | 2 +- source/libs/tdb/src/db/tdbDb.c | 6 +- source/util/src/tskiplist2.c | 1 - 9 files changed, 117 insertions(+), 31 deletions(-) diff --git a/include/util/tskiplist2.h b/include/util/tskiplist2.h index f3366dd574..83e6ad0868 100644 --- a/include/util/tskiplist2.h +++ b/include/util/tskiplist2.h @@ -26,6 +26,7 @@ extern "C" { typedef struct SSkipList2 SSkipList2; typedef struct SSLCursor SSLCursor; typedef struct SSLCfg SSLCfg; +typedef struct SSLNode SSLNode; typedef int32_t (*tslCmprFn)(const void *pKey1, int32_t nKey1, const void *pKey2, int32_t nKey2); diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index ca7e823d6b..5970fdbe92 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -103,8 +103,6 @@ typedef struct { #if 1 -// int metaCreateTable(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle); -int metaDropTable(SMeta* pMeta, tb_uid_t uid); SMSmaCursor* metaOpenSmaCursor(SMeta* pMeta, tb_uid_t uid); void metaCloseSmaCursor(SMSmaCursor* pSmaCur); int64_t metaSmaCursorNext(SMSmaCursor* pSmaCur); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 1c7f6034ee..5538a63b4b 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -46,13 +46,13 @@ extern "C" { #endif -typedef struct SVnodeInfo SVnodeInfo; -typedef struct SMeta SMeta; -typedef struct STsdb STsdb; -typedef struct STQ STQ; -typedef struct SVState SVState; -typedef struct SVBufPool SVBufPool; -typedef struct SQWorker SQHandle; +typedef struct SVnodeInfo SVnodeInfo; +typedef struct SMeta SMeta; +typedef struct STsdb STsdb; +typedef struct STQ STQ; +typedef struct SVState SVState; +typedef struct SVBufPool SVBufPool; +typedef struct SQWorker SQHandle; #define VNODE_META_DIR "meta" #define VNODE_TSDB_DIR "tsdb" @@ -77,6 +77,7 @@ int metaCommit(SMeta* pMeta); int metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq); int metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq); +int metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq); SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline); STSchema* metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver); int metaGetTableEntryByName(SMetaReader* pReader, const char* name); @@ -100,7 +101,7 @@ int32_t tsdbCreateTSma(STsdb* pTsdb, char* pMsg); int32_t tsdbInsertTSmaData(STsdb* pTsdb, int64_t indexUid, const char* msg); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, - uint64_t taskId); + uint64_t taskId); tsdbReaderT tsdbQueryCacheLastT(STsdb* tsdb, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, void* pMemRef); int32_t tsdbGetTableGroupFromIdListT(STsdb* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo); @@ -189,7 +190,6 @@ struct STbUidStore { #define TD_VID(PVNODE) (PVNODE)->config.vgId - static FORCE_INLINE bool vnodeIsRollup(SVnode* pVnode) { SRetention* pRetention = &(pVnode->config.tsdbCfg.retentions[0]); return (pRetention->freq > 0 && pRetention->keep > 0); diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 41ef231dcf..c09bafd859 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -98,7 +98,7 @@ int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) { // ret = tdbDbcOpen(pMeta->pCtbIdx, &pDbc1); // tdbDbcMoveTo(pDbc1, &pReq->suid, sizeof(pReq->suid), NULL /*cmpr*/, 0 /*TDB_FORWARD_SEARCH*/); // tdbDbcGet(pDbc1, &pKey, &kLen, &pVal, vLen); - // tdbDbcDrop(pDbc1); + // tdbDbcDelete(pDbc1); // // drop from pTagIdx // // drop from pTtlIdx // } @@ -166,18 +166,79 @@ _err: return -1; } -int metaDropTable(SMeta *pMeta, tb_uid_t uid) { -#if 0 - if (metaRemoveTableFromIdx(pMeta, uid) < 0) { - // TODO: handle error +int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) { + TDBC *pTbDbc = NULL; + TDBC *pUidIdxc = NULL; + TDBC *pNameIdxc = NULL; + const void *pData; + int nData; + tb_uid_t uid; + int64_t tver; + SMetaEntry me = {0}; + SCoder coder = {0}; + int c, ret; + + // search & delete the name idx + tdbDbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn); + ret = tdbDbcMoveTo(pNameIdxc, pReq->name, strlen(pReq->name) + 1, &c); + if (ret < 0 || c) { + tdbDbcClose(pNameIdxc); + terrno = TSDB_CODE_VND_TABLE_NOT_EXIST; return -1; } - if (metaRemoveTableFromIdx(pMeta, uid) < 0) { - // TODO + ret = tdbDbcGet(pNameIdxc, NULL, NULL, &pData, &nData); + if (ret < 0) { + ASSERT(0); return -1; } -#endif + + uid = *(tb_uid_t *)pData; + + tdbDbcDelete(pNameIdxc); + tdbDbcClose(pNameIdxc); + + // search & delete uid idx + tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); + ret = tdbDbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); + if (ret < 0 || c != 0) { + ASSERT(0); + return -1; + } + + ret = tdbDbcGet(pUidIdxc, NULL, NULL, &pData, &nData); + if (ret < 0) { + ASSERT(0); + return -1; + } + + tver = *(int64_t *)pData; + tdbDbcDelete(pUidIdxc); + tdbDbcClose(pUidIdxc); + + // search and get meta entry + tdbDbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); + ret = tdbDbcMoveTo(pTbDbc, &(STbDbKey){.uid = uid, .version = tver}, sizeof(STbDbKey), &c); + if (ret < 0 || c != 0) { + ASSERT(0); + return -1; + } + + ret = tdbDbcGet(pTbDbc, NULL, NULL, &pData, &nData); + if (ret < 0) { + ASSERT(0); + return -1; + } + + // tCoderInit(&coder, TD_LITTLE_ENDIAN, pData, nData, TD_DECODER); + // ret = metaDecodeEntry(&coder, &me); + // if (ret < 0) { + // ASSERT(0); + // return -1; + // } + + tCoderClear(&coder); + tdbDbcClose(pTbDbc); return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 91a470b118..878dd5fca4 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -445,14 +445,45 @@ static int vnodeProcessAlterTbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcM } static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { - SVDropTbReq req = {0}; - SVDropTbReq rsp = {0}; + SVDropTbBatchReq req = {0}; + SVDropTbBatchRsp rsp = {0}; + SCoder coder = {0}; + int ret; + + pRsp->msgType = TDMT_VND_CREATE_STB_RSP; + pRsp->pCont = NULL; + pRsp->contLen = 0; + pRsp->code = TSDB_CODE_SUCCESS; // decode req + tCoderInit(&coder, TD_LITTLE_ENDIAN, pReq, len, TD_DECODER); + ret = tDecodeSVDropTbBatchReq(&coder, &req); + if (ret < 0) { + terrno = TSDB_CODE_INVALID_MSG; + pRsp->code = terrno; + goto _exit; + } // process req + rsp.pArray = taosArrayInit(sizeof(SVDropTbRsp), req.nReqs); + for (int iReq = 0; iReq < req.nReqs; iReq++) { + SVDropTbReq *pDropTbReq = req.pReqs + iReq; + SVDropTbRsp dropTbRsp = {0}; - // return rsp + /* code */ + ret = metaDropTable(pVnode->pMeta, version, pDropTbReq); + if (ret < 0) { + dropTbRsp.code = TSDB_CODE_SUCCESS; + } else { + dropTbRsp.code = terrno; + } + + taosArrayPush(rsp.pArray, &dropTbRsp); + } + +_exit: + tCoderClear(&coder); + // encode rsp (TODO) return 0; } @@ -482,7 +513,7 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in } int32_t tsdbProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) { - if(!pReq) { + if (!pReq) { terrno = TSDB_CODE_INVALID_PTR; return TSDB_CODE_FAILED; } diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index dae085c672..cab242d5b3 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -58,7 +58,7 @@ int tdbDbcGet(TDBC *pDbc, const void **ppKey, int *pkLen, const void **ppVal, in int tdbDbcPut(TDBC *pDbc, const void *pKey, int keyLen, const void *pVal, int valLen); int tdbDbcUpdate(TDBC *pDbc, const void *pKey, int kLen, const void *pVal, int vLen); -int tdbDbcDrop(TDBC *pDbc); +int tdbDbcDelete(TDBC *pDbc); int tdbDbcNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen); // TXN diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 85cbae316d..ed7aed552a 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -1394,7 +1394,7 @@ int tdbBtcGet(SBTC *pBtc, const void **ppKey, int *kLen, const void **ppVal, int if (ppVal) { *ppVal = (void *)pBtc->coder.pVal; - *kLen = pBtc->coder.vLen; + *vLen = pBtc->coder.vLen; } return 0; diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index f9ef2e7901..3358aefec0 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -136,11 +136,7 @@ int tdbDbcUpdate(TDBC *pDbc, const void *pKey, int kLen, const void *pVal, int v return 0; } -int tdbDbcDrop(TDBC *pDbc) { - // TODO - ASSERT(0); - return 0; -} +int tdbDbcDelete(TDBC *pDbc) { return tdbBtcDelete(&pDbc->btc); } int tdbDbcNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen) { return tdbBtreeNext(&pDbc->btc, ppKey, kLen, ppVal, vLen); diff --git a/source/util/src/tskiplist2.c b/source/util/src/tskiplist2.c index e7391da285..77f5ed5051 100644 --- a/source/util/src/tskiplist2.c +++ b/source/util/src/tskiplist2.c @@ -16,7 +16,6 @@ #include "tskiplist2.h" -typedef struct SSLNode SSLNode; struct SSLNode { int8_t level; SSLNode *forwards[]; From e674de5bf9c905d03f067aacc2d2d13c01f4387f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 4 May 2022 12:25:39 +0800 Subject: [PATCH 25/48] refactor: adjust logs --- source/dnode/mnode/impl/src/mndTrans.c | 148 +++++++++--------- source/dnode/mnode/impl/test/trans/trans2.cpp | 2 + 2 files changed, 76 insertions(+), 74 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index bbd952b6d0..5a2f37429a 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -22,9 +22,9 @@ #include "mndSync.h" #include "mndUser.h" -#define MND_TRANS_VER_NUMBER 1 -#define MND_TRANS_ARRAY_SIZE 8 -#define MND_TRANS_RESERVE_SIZE 64 +#define TRANS_VER_NUMBER 1 +#define TRANS_ARRAY_SIZE 8 +#define TRANS_RESERVE_SIZE 64 static SSdbRaw *mndTransActionEncode(STrans *pTrans); static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw); @@ -84,7 +84,7 @@ void mndCleanupTrans(SMnode *pMnode) {} static SSdbRaw *mndTransActionEncode(STrans *pTrans) { terrno = TSDB_CODE_OUT_OF_MEMORY; - int32_t rawDataLen = sizeof(STrans) + MND_TRANS_RESERVE_SIZE; + int32_t rawDataLen = sizeof(STrans) + TRANS_RESERVE_SIZE; int32_t redoLogNum = taosArrayGetSize(pTrans->redoLogs); int32_t undoLogNum = taosArrayGetSize(pTrans->undoLogs); int32_t commitLogNum = taosArrayGetSize(pTrans->commitLogs); @@ -116,7 +116,7 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { rawDataLen += (sizeof(STransAction) + pAction->contLen); } - SSdbRaw *pRaw = sdbAllocRaw(SDB_TRANS, MND_TRANS_VER_NUMBER, rawDataLen); + SSdbRaw *pRaw = sdbAllocRaw(SDB_TRANS, TRANS_VER_NUMBER, rawDataLen); if (pRaw == NULL) { mError("trans:%d, failed to alloc raw since %s", pTrans->id, terrstr()); return NULL; @@ -182,7 +182,7 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_BINARY(pRaw, dataPos, pTrans->param, pTrans->paramLen, TRANS_ENCODE_OVER) } - SDB_SET_RESERVE(pRaw, dataPos, MND_TRANS_RESERVE_SIZE, TRANS_ENCODE_OVER) + SDB_SET_RESERVE(pRaw, dataPos, TRANS_RESERVE_SIZE, TRANS_ENCODE_OVER) SDB_SET_DATALEN(pRaw, dataPos, TRANS_ENCODE_OVER) terrno = 0; @@ -214,38 +214,38 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { int32_t dataPos = 0; STransAction action = {0}; - if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto TRANS_DECODE_OVER; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; - if (sver != MND_TRANS_VER_NUMBER) { + if (sver != TRANS_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; - goto TRANS_DECODE_OVER; + goto _OVER; } pRow = sdbAllocRow(sizeof(STrans)); - if (pRow == NULL) goto TRANS_DECODE_OVER; + if (pRow == NULL) goto _OVER; pTrans = sdbGetRowObj(pRow); - if (pTrans == NULL) goto TRANS_DECODE_OVER; + if (pTrans == NULL) goto _OVER; - SDB_GET_INT32(pRaw, dataPos, &pTrans->id, TRANS_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pTrans->id, _OVER) int16_t type = 0; int16_t policy = 0; int16_t stage = 0; - SDB_GET_INT16(pRaw, dataPos, &policy, TRANS_DECODE_OVER) - SDB_GET_INT16(pRaw, dataPos, &stage, TRANS_DECODE_OVER) - SDB_GET_INT16(pRaw, dataPos, &type, TRANS_DECODE_OVER) + SDB_GET_INT16(pRaw, dataPos, &policy, _OVER) + SDB_GET_INT16(pRaw, dataPos, &stage, _OVER) + SDB_GET_INT16(pRaw, dataPos, &type, _OVER) pTrans->policy = policy; pTrans->stage = stage; pTrans->transType = type; - SDB_GET_INT64(pRaw, dataPos, &pTrans->createdTime, TRANS_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pTrans->dbUid, TRANS_DECODE_OVER) - SDB_GET_BINARY(pRaw, dataPos, pTrans->dbname, TSDB_DB_FNAME_LEN, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &redoLogNum, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &undoLogNum, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &commitLogNum, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &redoActionNum, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &undoActionNum, TRANS_DECODE_OVER) + SDB_GET_INT64(pRaw, dataPos, &pTrans->createdTime, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pTrans->dbUid, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pTrans->dbname, TSDB_DB_FNAME_LEN, _OVER) + SDB_GET_INT32(pRaw, dataPos, &redoLogNum, _OVER) + SDB_GET_INT32(pRaw, dataPos, &undoLogNum, _OVER) + SDB_GET_INT32(pRaw, dataPos, &commitLogNum, _OVER) + SDB_GET_INT32(pRaw, dataPos, &redoActionNum, _OVER) + SDB_GET_INT32(pRaw, dataPos, &undoActionNum, _OVER) pTrans->redoLogs = taosArrayInit(redoLogNum, sizeof(void *)); pTrans->undoLogs = taosArrayInit(undoLogNum, sizeof(void *)); @@ -253,79 +253,79 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { pTrans->redoActions = taosArrayInit(redoActionNum, sizeof(STransAction)); pTrans->undoActions = taosArrayInit(undoActionNum, sizeof(STransAction)); - if (pTrans->redoLogs == NULL) goto TRANS_DECODE_OVER; - if (pTrans->undoLogs == NULL) goto TRANS_DECODE_OVER; - if (pTrans->commitLogs == NULL) goto TRANS_DECODE_OVER; - if (pTrans->redoActions == NULL) goto TRANS_DECODE_OVER; - if (pTrans->undoActions == NULL) goto TRANS_DECODE_OVER; + if (pTrans->redoLogs == NULL) goto _OVER; + if (pTrans->undoLogs == NULL) goto _OVER; + if (pTrans->commitLogs == NULL) goto _OVER; + if (pTrans->redoActions == NULL) goto _OVER; + if (pTrans->undoActions == NULL) goto _OVER; for (int32_t i = 0; i < redoLogNum; ++i) { - SDB_GET_INT32(pRaw, dataPos, &dataLen, TRANS_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) pData = taosMemoryMalloc(dataLen); - if (pData == NULL) goto TRANS_DECODE_OVER; + if (pData == NULL) goto _OVER; mTrace("raw:%p, is created", pData); - SDB_GET_BINARY(pRaw, dataPos, pData, dataLen, TRANS_DECODE_OVER); - if (taosArrayPush(pTrans->redoLogs, &pData) == NULL) goto TRANS_DECODE_OVER; + SDB_GET_BINARY(pRaw, dataPos, pData, dataLen, _OVER); + if (taosArrayPush(pTrans->redoLogs, &pData) == NULL) goto _OVER; pData = NULL; } for (int32_t i = 0; i < undoLogNum; ++i) { - SDB_GET_INT32(pRaw, dataPos, &dataLen, TRANS_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) pData = taosMemoryMalloc(dataLen); - if (pData == NULL) goto TRANS_DECODE_OVER; + if (pData == NULL) goto _OVER; mTrace("raw:%p, is created", pData); - SDB_GET_BINARY(pRaw, dataPos, pData, dataLen, TRANS_DECODE_OVER); - if (taosArrayPush(pTrans->undoLogs, &pData) == NULL) goto TRANS_DECODE_OVER; + SDB_GET_BINARY(pRaw, dataPos, pData, dataLen, _OVER); + if (taosArrayPush(pTrans->undoLogs, &pData) == NULL) goto _OVER; pData = NULL; } for (int32_t i = 0; i < commitLogNum; ++i) { - SDB_GET_INT32(pRaw, dataPos, &dataLen, TRANS_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) pData = taosMemoryMalloc(dataLen); - if (pData == NULL) goto TRANS_DECODE_OVER; + if (pData == NULL) goto _OVER; mTrace("raw:%p, is created", pData); - SDB_GET_BINARY(pRaw, dataPos, pData, dataLen, TRANS_DECODE_OVER); - if (taosArrayPush(pTrans->commitLogs, &pData) == NULL) goto TRANS_DECODE_OVER; + SDB_GET_BINARY(pRaw, dataPos, pData, dataLen, _OVER); + if (taosArrayPush(pTrans->commitLogs, &pData) == NULL) goto _OVER; pData = NULL; } for (int32_t i = 0; i < redoActionNum; ++i) { - SDB_GET_BINARY(pRaw, dataPos, (void *)&action.epSet, sizeof(SEpSet), TRANS_DECODE_OVER); - SDB_GET_INT16(pRaw, dataPos, &action.msgType, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &action.contLen, TRANS_DECODE_OVER) + SDB_GET_BINARY(pRaw, dataPos, (void *)&action.epSet, sizeof(SEpSet), _OVER); + SDB_GET_INT16(pRaw, dataPos, &action.msgType, _OVER) + SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) + SDB_GET_INT32(pRaw, dataPos, &action.contLen, _OVER) action.pCont = taosMemoryMalloc(action.contLen); - if (action.pCont == NULL) goto TRANS_DECODE_OVER; - SDB_GET_BINARY(pRaw, dataPos, action.pCont, action.contLen, TRANS_DECODE_OVER); - if (taosArrayPush(pTrans->redoActions, &action) == NULL) goto TRANS_DECODE_OVER; + if (action.pCont == NULL) goto _OVER; + SDB_GET_BINARY(pRaw, dataPos, action.pCont, action.contLen, _OVER); + if (taosArrayPush(pTrans->redoActions, &action) == NULL) goto _OVER; action.pCont = NULL; } for (int32_t i = 0; i < undoActionNum; ++i) { - SDB_GET_BINARY(pRaw, dataPos, (void *)&action.epSet, sizeof(SEpSet), TRANS_DECODE_OVER); - SDB_GET_INT16(pRaw, dataPos, &action.msgType, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &action.contLen, TRANS_DECODE_OVER) + SDB_GET_BINARY(pRaw, dataPos, (void *)&action.epSet, sizeof(SEpSet), _OVER); + SDB_GET_INT16(pRaw, dataPos, &action.msgType, _OVER) + SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) + SDB_GET_INT32(pRaw, dataPos, &action.contLen, _OVER) action.pCont = taosMemoryMalloc(action.contLen); - if (action.pCont == NULL) goto TRANS_DECODE_OVER; - SDB_GET_BINARY(pRaw, dataPos, action.pCont, action.contLen, TRANS_DECODE_OVER); - if (taosArrayPush(pTrans->undoActions, &action) == NULL) goto TRANS_DECODE_OVER; + if (action.pCont == NULL) goto _OVER; + SDB_GET_BINARY(pRaw, dataPos, action.pCont, action.contLen, _OVER); + if (taosArrayPush(pTrans->undoActions, &action) == NULL) goto _OVER; action.pCont = NULL; } - SDB_GET_INT32(pRaw, dataPos, &pTrans->startFunc, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pTrans->stopFunc, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pTrans->paramLen, TRANS_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pTrans->startFunc, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pTrans->stopFunc, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pTrans->paramLen, _OVER) if (pTrans->paramLen != 0) { pTrans->param = taosMemoryMalloc(pTrans->paramLen); - SDB_GET_BINARY(pRaw, dataPos, pTrans->param, pTrans->paramLen, TRANS_DECODE_OVER); + SDB_GET_BINARY(pRaw, dataPos, pTrans->param, pTrans->paramLen, _OVER); } - SDB_GET_RESERVE(pRaw, dataPos, MND_TRANS_RESERVE_SIZE, TRANS_DECODE_OVER) + SDB_GET_RESERVE(pRaw, dataPos, TRANS_RESERVE_SIZE, _OVER) terrno = 0; -TRANS_DECODE_OVER: +_OVER: if (terrno != 0) { mError("trans:%d, failed to parse from raw:%p since %s", pTrans->id, pRaw, terrstr()); mndTransDropData(pTrans); @@ -545,11 +545,11 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnType type, const S pTrans->rpcHandle = pReq->handle; pTrans->rpcAHandle = pReq->ahandle; pTrans->rpcRefId = pReq->refId; - pTrans->redoLogs = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(void *)); - pTrans->undoLogs = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(void *)); - pTrans->commitLogs = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(void *)); - pTrans->redoActions = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(STransAction)); - pTrans->undoActions = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(STransAction)); + pTrans->redoLogs = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(void *)); + pTrans->undoLogs = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(void *)); + pTrans->commitLogs = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(void *)); + pTrans->redoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); + pTrans->undoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); if (pTrans->redoLogs == NULL || pTrans->undoLogs == NULL || pTrans->commitLogs == NULL || pTrans->redoActions == NULL || pTrans->undoActions == NULL) { @@ -855,7 +855,7 @@ void mndTransProcessRsp(SNodeMsg *pRsp) { STrans *pTrans = mndAcquireTrans(pMnode, transId); if (pTrans == NULL) { mError("trans:%d, failed to get transId from vnode rsp since %s", transId, terrstr()); - goto HANDLE_ACTION_RSP_OVER; + goto _OVER; } SArray *pArray = NULL; @@ -865,18 +865,18 @@ void mndTransProcessRsp(SNodeMsg *pRsp) { pArray = pTrans->undoActions; } else { mError("trans:%d, invalid trans stage:%d while recv action rsp", pTrans->id, pTrans->stage); - goto HANDLE_ACTION_RSP_OVER; + goto _OVER; } if (pArray == NULL) { mError("trans:%d, invalid trans stage:%d", transId, pTrans->stage); - goto HANDLE_ACTION_RSP_OVER; + goto _OVER; } int32_t actionNum = taosArrayGetSize(pTrans->redoActions); if (action < 0 || action >= actionNum) { mError("trans:%d, invalid action:%d", transId, action); - goto HANDLE_ACTION_RSP_OVER; + goto _OVER; } STransAction *pAction = taosArrayGet(pArray, action); @@ -892,7 +892,7 @@ void mndTransProcessRsp(SNodeMsg *pRsp) { pAction->acceptableCode); mndTransExecute(pMnode, pTrans); -HANDLE_ACTION_RSP_OVER: +_OVER: mndReleaseTrans(pMnode, pTrans); } @@ -1281,19 +1281,19 @@ static int32_t mndProcessKillTransReq(SNodeMsg *pReq) { if (tDeserializeSKillTransReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &killReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto KILL_OVER; + goto _OVER; } mInfo("trans:%d, start to kill", killReq.transId); pUser = mndAcquireUser(pMnode, pReq->user); if (pUser == NULL) { - goto KILL_OVER; + goto _OVER; } if (!pUser->superUser) { terrno = TSDB_CODE_MND_NO_RIGHTS; - goto KILL_OVER; + goto _OVER; } pTrans = mndAcquireTrans(pMnode, killReq.transId); @@ -1305,7 +1305,7 @@ static int32_t mndProcessKillTransReq(SNodeMsg *pReq) { code = mndKillTrans(pMnode, pTrans); -KILL_OVER: +_OVER: if (code != 0) { mError("trans:%d, failed to kill since %s", killReq.transId, terrstr()); return -1; diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp index ef80aa14eb..abab24723f 100644 --- a/source/dnode/mnode/impl/test/trans/trans2.cpp +++ b/source/dnode/mnode/impl/test/trans/trans2.cpp @@ -57,7 +57,9 @@ class MndTestTrans2 : public ::testing::Test { opt.replicas[0].port = 9040; strcpy(opt.replicas[0].fqdn, "localhost"); opt.msgCb = msgCb; + const char *mnodepath = "/tmp/mnode_test_trans"; + taosRemoveDir(mnodepath); pMnode = mndOpen(mnodepath, &opt); } From 5ed7b44b165ded8d3b391fd748e64bd2f4b2c55a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 4 May 2022 12:35:05 +0800 Subject: [PATCH 26/48] enh(index): support index filter --- include/libs/index/index.h | 12 +++++++++++- source/libs/executor/src/indexoperator.c | 14 ++++++++------ source/libs/index/inc/indexCache.h | 1 + source/libs/index/inc/indexInt.h | 1 - source/libs/index/src/index.c | 1 - source/libs/index/src/indexCache.c | 23 ++++++++++++++++++++++- source/libs/index/src/indexTfile.c | 23 ++++++++++++++++++++++- 7 files changed, 64 insertions(+), 11 deletions(-) diff --git a/include/libs/index/index.h b/include/libs/index/index.h index bf260d4899..c94d75338a 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -47,7 +47,17 @@ typedef enum { } SIndexOperOnColumn; typedef enum { MUST = 0, SHOULD, NOT } EIndexOperatorType; -typedef enum { QUERY_TERM = 0, QUERY_PREFIX, QUERY_SUFFIX, QUERY_REGEX, QUERY_RANGE } EIndexQueryType; +typedef enum { + QUERY_TERM = 0, + QUERY_PREFIX, + QUERY_SUFFIX, + QUERY_REGEX, + QUERY_LESS_THAN, + QUERY_LESS_EQUAL, + QUERY_GREATER_THAN, + QUERY_GREATER_EQUAL, + QUERY_RANGE +} EIndexQueryType; /* * create multi query diff --git a/source/libs/executor/src/indexoperator.c b/source/libs/executor/src/indexoperator.c index 0f075c5de2..3b62bdd664 100644 --- a/source/libs/executor/src/indexoperator.c +++ b/source/libs/executor/src/indexoperator.c @@ -63,9 +63,14 @@ typedef struct SIFParam { } SIFParam; static int32_t sifGetFuncFromSql(EOperatorType src, EIndexQueryType *dst) { - if (src == OP_TYPE_GREATER_THAN || src == OP_TYPE_GREATER_EQUAL || src == OP_TYPE_LOWER_THAN || - src == OP_TYPE_LOWER_EQUAL) { - *dst = QUERY_RANGE; + if (src == OP_TYPE_GREATER_THAN) { + *dst = QUERY_GREATER_THAN; + } else if (src == OP_TYPE_GREATER_EQUAL) { + *dst = QUERY_GREATER_EQUAL; + } else if (src == OP_TYPE_LOWER_THAN) { + *dst = QUERY_LESS_THAN; + } else if (src == OP_TYPE_LOWER_EQUAL) { + *dst = QUERY_LESS_EQUAL; } else if (src == OP_TYPE_EQUAL) { *dst = QUERY_TERM; } else if (src == OP_TYPE_LIKE || src == OP_TYPE_MATCH || src == OP_TYPE_NMATCH) { @@ -249,9 +254,6 @@ static int32_t sifExecFunction(SFunctionNode *node, SIFCtx *ctx, SIFParam *outpu static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) { SIndexTerm *tm = indexTermCreate(left->suid, DEFAULT, operType, left->colValType, left->colName, strlen(left->colName), right->condValue, strlen(right->condValue)); - if (operType == OP_TYPE_LOWER_EQUAL || operType == OP_TYPE_GREATER_EQUAL || operType == OP_TYPE_GREATER_THAN || - operType == OP_TYPE_LOWER_THAN) { - } if (tm == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } diff --git a/source/libs/index/inc/indexCache.h b/source/libs/index/inc/indexCache.h index 3ea986ad48..44355db7a7 100644 --- a/source/libs/index/inc/indexCache.h +++ b/source/libs/index/inc/indexCache.h @@ -56,6 +56,7 @@ typedef struct CacheTerm { int8_t colType; SIndexOperOnColumn operaType; + int8_t qType; // query type } CacheTerm; // diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 62ddf85985..04817c4df3 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -86,7 +86,6 @@ typedef struct SIndexTerm { int32_t nColName; char* colVal; int32_t nColVal; - int8_t qType; // just use for range } SIndexTerm; typedef struct SIndexTermQuery { diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index ee77aa92e9..83b5025ad0 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -262,7 +262,6 @@ SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, int8_t queryT tm->colVal = (char*)taosMemoryCalloc(1, nColVal + 1); memcpy(tm->colVal, colVal, nColVal); tm->nColVal = nColVal; - tm->qType = queryType; return tm; } diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 78368981b3..ea3cbf5658 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -38,10 +38,15 @@ static int32_t cacheSearchTerm(void* cache, CacheTerm* ct, SIdxTempResult* tr, S static int32_t cacheSearchPrefix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); static int32_t cacheSearchSuffix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); static int32_t cacheSearchRegex(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchLessThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchLessEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchGreaterThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchGreaterEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); static int32_t cacheSearchRange(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); static int32_t (*cacheSearch[])(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) = { - cacheSearchTerm, cacheSearchPrefix, cacheSearchSuffix, cacheSearchRegex, cacheSearchRange}; + cacheSearchTerm, cacheSearchPrefix, cacheSearchSuffix, cacheSearchRegex, cacheSearchLessThan, + cacheSearchLessEqual, cacheSearchGreaterThan, cacheSearchGreaterEqual, cacheSearchRange}; static void doMergeWork(SSchedMsg* msg); static bool indexCacheIteratorNext(Iterate* itera); @@ -88,6 +93,22 @@ static int32_t cacheSearchRegex(void* cache, CacheTerm* ct, SIdxTempResult* tr, // impl later return 0; } +static int32_t cacheSearchLessThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { + // impl later + return 0; +} +static int32_t cacheSearchLessEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { + // impl later + return 0; +} +static int32_t cacheSearchGreaterThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { + // impl later + return 0; +} +static int32_t cacheSearchGreaterEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { + // impl later + return 0; +} static int32_t cacheSearchRange(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { // impl later return 0; diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 7072baf574..327a06084c 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -64,10 +64,15 @@ static int32_t tfSearchTerm(void* reader, SIndexTerm* tem, SIdxTempResult* tr); static int32_t tfSearchPrefix(void* reader, SIndexTerm* tem, SIdxTempResult* tr); static int32_t tfSearchSuffix(void* reader, SIndexTerm* tem, SIdxTempResult* tr); static int32_t tfSearchRegex(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchLessThan(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchLessEqual(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchGreaterThan(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchGreaterEqual(void* reader, SIndexTerm* tem, SIdxTempResult* tr); static int32_t tfSearchRange(void* reader, SIndexTerm* tem, SIdxTempResult* tr); static int32_t (*tfSearch[])(void* reader, SIndexTerm* tem, SIdxTempResult* tr) = { - tfSearchTerm, tfSearchPrefix, tfSearchSuffix, tfSearchRegex, tfSearchRange}; + tfSearchTerm, tfSearchPrefix, tfSearchSuffix, tfSearchRegex, tfSearchLessThan, + tfSearchLessEqual, tfSearchGreaterThan, tfSearchGreaterEqual, tfSearchRange}; TFileCache* tfileCacheCreate(const char* path) { TFileCache* tcache = taosMemoryCalloc(1, sizeof(TFileCache)); @@ -299,6 +304,22 @@ static int32_t tfSearchRegex(void* reader, SIndexTerm* tem, SIdxTempResult* tr) fstSliceDestroy(&key); return 0; } +static int32_t tfSearchLessThan(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + // impl later + return 0; +} +static int32_t tfSearchLessEqual(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + // impl later + return 0; +} +static int32_t tfSearchGreaterThan(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + // impl later + return 0; +} +static int32_t tfSearchGreaterEqual(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + // impl later + return 0; +} static int32_t tfSearchRange(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON); int ret = 0; From d5e578cf163b2b6477642744542314cb3bc71c9c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 4 May 2022 06:16:29 +0000 Subject: [PATCH 27/48] implement drop table --- source/dnode/vnode/src/meta/metaTable.c | 55 ++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index c09bafd859..7ddac9337d 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -176,6 +176,9 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) { int64_t tver; SMetaEntry me = {0}; SCoder coder = {0}; + int8_t type; + int64_t ctime; + tb_uid_t suid; int c, ret; // search & delete the name idx @@ -230,16 +233,56 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) { return -1; } - // tCoderInit(&coder, TD_LITTLE_ENDIAN, pData, nData, TD_DECODER); - // ret = metaDecodeEntry(&coder, &me); - // if (ret < 0) { - // ASSERT(0); - // return -1; - // } + // decode entry + void *pDataCopy = taosMemoryMalloc(nData); // remove the copy (todo) + memcpy(pDataCopy, pData, nData); + tCoderInit(&coder, TD_LITTLE_ENDIAN, pDataCopy, nData, TD_DECODER); + ret = metaDecodeEntry(&coder, &me); + if (ret < 0) { + ASSERT(0); + return -1; + } + type = me.type; + if (type == TSDB_CHILD_TABLE) { + ctime = me.ctbEntry.ctime; + suid = me.ctbEntry.suid; + } else if (type == TSDB_NORMAL_TABLE) { + ctime = me.ntbEntry.ctime; + suid = 0; + } else { + ASSERT(0); + } + + taosMemoryFree(pDataCopy); tCoderClear(&coder); tdbDbcClose(pTbDbc); + if (type == TSDB_CHILD_TABLE) { + // remove the pCtbIdx + TDBC *pCtbIdxc = NULL; + tdbDbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn); + + ret = tdbDbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = uid}, sizeof(SCtbIdxKey), &c); + if (ret < 0 || c != 0) { + ASSERT(0); + return -1; + } + + tdbDbcDelete(pCtbIdxc); + tdbDbcClose(pCtbIdxc); + + // remove tags from pTagIdx (todo) + } else if (type == TSDB_NORMAL_TABLE) { + // remove from pSkmDb + } else { + ASSERT(0); + } + + // remove from ttl (todo) + if (ctime > 0) { + } + return 0; } From 9d2df3531670cb37266aa73fb71ae9a7cf5ff105 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 4 May 2022 07:07:34 +0000 Subject: [PATCH 28/48] drop super table --- source/dnode/vnode/src/meta/metaTable.c | 75 +++++++++++++++---------- source/dnode/vnode/src/vnd/vnodeSvr.c | 8 +-- 2 files changed, 50 insertions(+), 33 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 7ddac9337d..397e074061 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -72,44 +72,61 @@ _err: } int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) { - SMetaReader mr = {0}; + TDBC *pNameIdxc = NULL; + TDBC *pUidIdxc = NULL; + TDBC *pCtbIdxc = NULL; + SCtbIdxKey *pCtbIdxKey; + const void *pKey = NULL; + int nKey; + const void *pData = NULL; + int nData; + int c, ret; - // validate req - metaReaderInit(&mr, pMeta, 0); - if (metaGetTableEntryByUid(&mr, pReq->suid) < 0) { - terrno = TSDB_CODE_VND_TABLE_NOT_EXIST; + // prepare uid idx cursor + tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); + ret = tdbDbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c); + if (ret < 0 || c != 0) { + terrno = TSDB_CODE_VND_TB_NOT_EXIST; + tdbDbcClose(pUidIdxc); goto _err; } - // do drop - // drop from pTbDb - // drop from pSkmDb - // drop from pUidIdx - // drop from pNameIdx - // { - // TDBC *pDbc1 = NULL; - // void *pKey = NULL; - // void *pVal = NULL; - // int kLen = 0; - // int vLen = 0; - // int ret = 0; + // prepare name idx cursor + tdbDbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn); + ret = tdbDbcMoveTo(pNameIdxc, pReq->name, strlen(pReq->name) + 1, &c); + if (ret < 0 || c != 0) { + ASSERT(0); + } - // // drop from pCtbIdx - // ret = tdbDbcOpen(pMeta->pCtbIdx, &pDbc1); - // tdbDbcMoveTo(pDbc1, &pReq->suid, sizeof(pReq->suid), NULL /*cmpr*/, 0 /*TDB_FORWARD_SEARCH*/); - // tdbDbcGet(pDbc1, &pKey, &kLen, &pVal, vLen); - // tdbDbcDelete(pDbc1); - // // drop from pTagIdx - // // drop from pTtlIdx - // } + tdbDbcDelete(pUidIdxc); + tdbDbcDelete(pNameIdxc); + tdbDbcClose(pUidIdxc); + tdbDbcClose(pNameIdxc); - // clear and return - metaReaderClear(&mr); - metaError("vgId:%d super table %s uid:%" PRId64 " is dropped", TD_VID(pMeta->pVnode), pReq->name, pReq->suid); + // loop to drop each child table + tdbDbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn); + ret = tdbDbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = pReq->suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); + if (ret < 0 || (c < 0 && tdbDbcMoveToNext(pCtbIdxc) < 0)) { + tdbDbcClose(pCtbIdxc); + goto _exit; + } + + for (;;) { + tdbDbcGet(pCtbIdxc, &pKey, &nKey, NULL, NULL); + pCtbIdxKey = (SCtbIdxKey *)pKey; + + if (pCtbIdxKey->suid > pReq->suid) break; + + // drop the child table (TODO) + + if (tdbDbcMoveToNext(pCtbIdxc) < 0) break; + } + +_exit: + metaDebug("vgId:%d super table %s uid:%" PRId64 " is dropped", TD_VID(pMeta->pVnode), pReq->name, pReq->suid); return 0; _err: - metaReaderClear(&mr); metaError("vgId:%d failed to drop super table %s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, tstrerror(terrno)); return -1; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 878dd5fca4..7d02840600 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -426,10 +426,10 @@ static int vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, i } // process request - // if (metaDropSTable(pVnode->pMeta, version, &req) < 0) { - // rcode = terrno; - // goto _exit; - // } + if (metaDropSTable(pVnode->pMeta, version, &req) < 0) { + rcode = terrno; + goto _exit; + } // return rsp _exit: From 8fda7b64dcb531ad8240f1a13f7f9c3615459143 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 4 May 2022 15:11:40 +0800 Subject: [PATCH 29/48] refactor: adjust transaction unitest --- include/util/taoserror.h | 2 +- source/dnode/mnode/impl/inc/mndDef.h | 5 +++- source/dnode/mnode/impl/src/mndTrans.c | 38 +++++++++++++------------- source/util/src/terror.c | 2 +- 4 files changed, 25 insertions(+), 22 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 27af7eb27e..b8baf99552 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -264,7 +264,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_TRANS_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03D0) #define TSDB_CODE_MND_TRANS_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03D1) #define TSDB_CODE_MND_TRANS_INVALID_STAGE TAOS_DEF_ERROR_CODE(0, 0x03D2) -#define TSDB_CODE_MND_TRANS_CANT_PARALLEL TAOS_DEF_ERROR_CODE(0, 0x03D4) +#define TSDB_CODE_MND_TRANS_CAN_NOT_PARALLEL TAOS_DEF_ERROR_CODE(0, 0x03D4) // mnode-mq #define TSDB_CODE_MND_TOPIC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E0) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 263fd0bad5..e48a9b83c0 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -91,10 +91,12 @@ typedef enum { TRN_TYPE_CONSUMER_LOST = 1022, TRN_TYPE_CONSUMER_RECOVER = 1023, TRN_TYPE_BASIC_SCOPE_END, + TRN_TYPE_GLOBAL_SCOPE = 2000, TRN_TYPE_CREATE_DNODE = 2001, TRN_TYPE_DROP_DNODE = 2002, TRN_TYPE_GLOBAL_SCOPE_END, + TRN_TYPE_DB_SCOPE = 3000, TRN_TYPE_CREATE_DB = 3001, TRN_TYPE_ALTER_DB = 3002, @@ -102,6 +104,7 @@ typedef enum { TRN_TYPE_SPLIT_VGROUP = 3004, TRN_TYPE_MERGE_VGROUP = 3015, TRN_TYPE_DB_SCOPE_END, + TRN_TYPE_STB_SCOPE = 4000, TRN_TYPE_CREATE_STB = 4001, TRN_TYPE_ALTER_STB = 4002, @@ -131,7 +134,7 @@ typedef struct { int32_t id; ETrnStage stage; ETrnPolicy policy; - ETrnType transType; + ETrnType type; int32_t code; int32_t failedTimes; void* rpcHandle; diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 5a2f37429a..eb946e9f53 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -126,7 +126,7 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_INT32(pRaw, dataPos, pTrans->id, TRANS_ENCODE_OVER) SDB_SET_INT16(pRaw, dataPos, pTrans->policy, TRANS_ENCODE_OVER) SDB_SET_INT16(pRaw, dataPos, pTrans->stage, TRANS_ENCODE_OVER) - SDB_SET_INT16(pRaw, dataPos, pTrans->transType, TRANS_ENCODE_OVER) + SDB_SET_INT16(pRaw, dataPos, pTrans->type, TRANS_ENCODE_OVER) SDB_SET_INT64(pRaw, dataPos, pTrans->createdTime, TRANS_ENCODE_OVER) SDB_SET_INT64(pRaw, dataPos, pTrans->dbUid, TRANS_ENCODE_OVER) SDB_SET_BINARY(pRaw, dataPos, pTrans->dbname, TSDB_DB_FNAME_LEN, TRANS_ENCODE_OVER) @@ -237,7 +237,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT16(pRaw, dataPos, &type, _OVER) pTrans->policy = policy; pTrans->stage = stage; - pTrans->transType = type; + pTrans->type = type; SDB_GET_INT64(pRaw, dataPos, &pTrans->createdTime, _OVER) SDB_GET_INT64(pRaw, dataPos, &pTrans->dbUid, _OVER) SDB_GET_BINARY(pRaw, dataPos, pTrans->dbname, TSDB_DB_FNAME_LEN, _OVER) @@ -540,7 +540,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnType type, const S pTrans->id = sdbGetMaxId(pMnode->pSdb, SDB_TRANS); pTrans->stage = TRN_STAGE_PREPARE; pTrans->policy = policy; - pTrans->transType = type; + pTrans->type = type; pTrans->createdTime = taosGetTimestampMs(); pTrans->rpcHandle = pReq->handle; pTrans->rpcAHandle = pReq->ahandle; @@ -592,7 +592,7 @@ void mndTransDrop(STrans *pTrans) { static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw) { if (pArray == NULL || pRaw == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + terrno = TSDB_CODE_INVALID_PARA; return -1; } @@ -689,12 +689,12 @@ static bool mndIsStbTrans(STrans *pTrans) { return pTrans->stage > TRN_TYPE_STB_SCOPE && pTrans->stage < TRN_TYPE_STB_SCOPE_END; } -static int32_t mndCheckTransCanBeStartedInParallel(SMnode *pMnode, STrans *pNewTrans) { - if (mndIsBasicTrans(pNewTrans)) return 0; - +static bool mndCheckTransCanParallel(SMnode *pMnode, STrans *pNewTrans) { STrans *pTrans = NULL; void *pIter = NULL; - int32_t code = 0; + bool canParallel = true; + + if (mndIsBasicTrans(pNewTrans)) return canParallel; while (1) { pIter = sdbFetch(pMnode->pSdb, SDB_TRANS, pIter, (void **)&pTrans); @@ -703,7 +703,7 @@ static int32_t mndCheckTransCanBeStartedInParallel(SMnode *pMnode, STrans *pNewT if (mndIsGlobalTrans(pNewTrans)) { if (mndIsDbTrans(pTrans) || mndIsStbTrans(pTrans)) { mError("trans:%d, can't execute since trans:%d in progress db:%s", pNewTrans->id, pTrans->id, pTrans->dbname); - code = -1; + canParallel = false; break; } } @@ -712,13 +712,13 @@ static int32_t mndCheckTransCanBeStartedInParallel(SMnode *pMnode, STrans *pNewT if (mndIsBasicTrans(pTrans)) continue; if (mndIsGlobalTrans(pTrans)) { mError("trans:%d, can't execute since trans:%d in progress", pNewTrans->id, pTrans->id); - code = -1; + canParallel = false; break; } if (mndIsDbTrans(pTrans) || mndIsStbTrans(pTrans)) { if (pNewTrans->dbUid == pTrans->dbUid) { mError("trans:%d, can't execute since trans:%d in progress db:%s", pNewTrans->id, pTrans->id, pTrans->dbname); - code = -1; + canParallel = false; break; } } @@ -728,13 +728,13 @@ static int32_t mndCheckTransCanBeStartedInParallel(SMnode *pMnode, STrans *pNewT if (mndIsBasicTrans(pTrans)) continue; if (mndIsGlobalTrans(pTrans)) { mError("trans:%d, can't execute since trans:%d in progress", pNewTrans->id, pTrans->id); - code = -1; + canParallel = false; break; } if (mndIsDbTrans(pTrans)) { if (pNewTrans->dbUid == pTrans->dbUid) { mError("trans:%d, can't execute since trans:%d in progress db:%s", pNewTrans->id, pTrans->id, pTrans->dbname); - code = -1; + canParallel = false; break; } } @@ -746,12 +746,12 @@ static int32_t mndCheckTransCanBeStartedInParallel(SMnode *pMnode, STrans *pNewT sdbCancelFetch(pMnode->pSdb, pIter); sdbRelease(pMnode->pSdb, pTrans); - return code; + return canParallel; } int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { - if (mndCheckTransCanBeStartedInParallel(pMnode, pTrans) != 0) { - terrno = TSDB_CODE_MND_TRANS_CANT_PARALLEL; + if (!mndCheckTransCanParallel(pMnode, pTrans)) { + terrno = TSDB_CODE_MND_TRANS_CAN_NOT_PARALLEL; mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); return -1; } @@ -1360,10 +1360,10 @@ static int32_t mndRetrieveTrans(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pB pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)dbname, false); - char transType[TSDB_TRANS_TYPE_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_WITH_MAXSIZE_TO_VARSTR(dbname, mndTransType(pTrans->transType), pShow->pMeta->pSchemas[cols].bytes); + char type[TSDB_TRANS_TYPE_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(dbname, mndTransType(pTrans->type), pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)transType, false); + colDataAppend(pColInfo, numOfRows, (const char *)type, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pTrans->lastExecTime, false); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 709efa3125..bd787d50b0 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -271,7 +271,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_RETRIEVE, "Invalid func retrieve TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_ALREADY_EXIST, "Transaction already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_NOT_EXIST, "Transaction not exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_INVALID_STAGE, "Invalid stage to kill") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CANT_PARALLEL, "Invalid stage to kill") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CAN_NOT_PARALLEL, "Conflicting transaction not completed") // mnode-mq TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_ALREADY_EXIST, "Topic already exists") From c734e2d9a19ed6033ff02fd0a93b9d830e6a261e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 4 May 2022 07:15:50 +0000 Subject: [PATCH 30/48] make case pass --- source/dnode/vnode/src/vnd/vnodeSvr.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 7d02840600..878dd5fca4 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -426,10 +426,10 @@ static int vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, i } // process request - if (metaDropSTable(pVnode->pMeta, version, &req) < 0) { - rcode = terrno; - goto _exit; - } + // if (metaDropSTable(pVnode->pMeta, version, &req) < 0) { + // rcode = terrno; + // goto _exit; + // } // return rsp _exit: From 1dd08f30e6250aee6871c17504c9d9cd5515d99a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 4 May 2022 15:47:01 +0800 Subject: [PATCH 31/48] fix: only write three sages prepare, rollback, and commit of trans to sdb --- source/dnode/mnode/impl/inc/mndDef.h | 10 +-- source/dnode/mnode/impl/src/mndTrans.c | 105 ++++++++++++++----------- 2 files changed, 62 insertions(+), 53 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index e48a9b83c0..a3c143df41 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -57,11 +57,11 @@ typedef enum { TRN_STAGE_PREPARE = 0, TRN_STAGE_REDO_LOG = 1, TRN_STAGE_REDO_ACTION = 2, - TRN_STAGE_COMMIT = 3, - TRN_STAGE_COMMIT_LOG = 4, - TRN_STAGE_UNDO_ACTION = 5, - TRN_STAGE_UNDO_LOG = 6, - TRN_STAGE_ROLLBACK = 7, + TRN_STAGE_ROLLBACK = 3, + TRN_STAGE_UNDO_ACTION = 4, + TRN_STAGE_UNDO_LOG = 5, + TRN_STAGE_COMMIT = 6, + TRN_STAGE_COMMIT_LOG = 7, TRN_STAGE_FINISHED = 8 } ETrnStage; diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index eb946e9f53..f3d7bf8c66 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -123,71 +123,82 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { } int32_t dataPos = 0; - SDB_SET_INT32(pRaw, dataPos, pTrans->id, TRANS_ENCODE_OVER) - SDB_SET_INT16(pRaw, dataPos, pTrans->policy, TRANS_ENCODE_OVER) - SDB_SET_INT16(pRaw, dataPos, pTrans->stage, TRANS_ENCODE_OVER) - SDB_SET_INT16(pRaw, dataPos, pTrans->type, TRANS_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pTrans->createdTime, TRANS_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pTrans->dbUid, TRANS_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, pTrans->dbname, TSDB_DB_FNAME_LEN, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, redoLogNum, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, undoLogNum, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, commitLogNum, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, redoActionNum, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, undoActionNum, TRANS_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pTrans->id, _OVER) + + ETrnStage stage = pTrans->stage; + if (stage == TRN_STAGE_REDO_LOG || stage == TRN_STAGE_REDO_ACTION) { + stage = TRN_STAGE_PREPARE; + } else if (stage == TRN_STAGE_UNDO_ACTION || stage == TRN_STAGE_UNDO_LOG) { + stage = TRN_STAGE_ROLLBACK; + } else if (stage == TRN_STAGE_COMMIT_LOG || stage == TRN_STAGE_FINISHED) { + stage = TRN_STAGE_COMMIT; + } else { + } + + SDB_SET_INT16(pRaw, dataPos, stage, _OVER) + SDB_SET_INT16(pRaw, dataPos, pTrans->policy, _OVER) + SDB_SET_INT16(pRaw, dataPos, pTrans->type, _OVER) + SDB_SET_INT64(pRaw, dataPos, pTrans->createdTime, _OVER) + SDB_SET_INT64(pRaw, dataPos, pTrans->dbUid, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pTrans->dbname, TSDB_DB_FNAME_LEN, _OVER) + SDB_SET_INT32(pRaw, dataPos, redoLogNum, _OVER) + SDB_SET_INT32(pRaw, dataPos, undoLogNum, _OVER) + SDB_SET_INT32(pRaw, dataPos, commitLogNum, _OVER) + SDB_SET_INT32(pRaw, dataPos, redoActionNum, _OVER) + SDB_SET_INT32(pRaw, dataPos, undoActionNum, _OVER) for (int32_t i = 0; i < redoLogNum; ++i) { SSdbRaw *pTmp = taosArrayGetP(pTrans->redoLogs, i); int32_t len = sdbGetRawTotalSize(pTmp); - SDB_SET_INT32(pRaw, dataPos, len, TRANS_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len, TRANS_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, len, _OVER) + SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len, _OVER) } for (int32_t i = 0; i < undoLogNum; ++i) { SSdbRaw *pTmp = taosArrayGetP(pTrans->undoLogs, i); int32_t len = sdbGetRawTotalSize(pTmp); - SDB_SET_INT32(pRaw, dataPos, len, TRANS_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len, TRANS_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, len, _OVER) + SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len, _OVER) } for (int32_t i = 0; i < commitLogNum; ++i) { SSdbRaw *pTmp = taosArrayGetP(pTrans->commitLogs, i); int32_t len = sdbGetRawTotalSize(pTmp); - SDB_SET_INT32(pRaw, dataPos, len, TRANS_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len, TRANS_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, len, _OVER) + SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len, _OVER) } for (int32_t i = 0; i < redoActionNum; ++i) { STransAction *pAction = taosArrayGet(pTrans->redoActions, i); - SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet), TRANS_ENCODE_OVER) - SDB_SET_INT16(pRaw, dataPos, pAction->msgType, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->contLen, TRANS_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, pAction->pCont, pAction->contLen, TRANS_ENCODE_OVER) + SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet), _OVER) + SDB_SET_INT16(pRaw, dataPos, pAction->msgType, _OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->contLen, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pAction->pCont, pAction->contLen, _OVER) } for (int32_t i = 0; i < undoActionNum; ++i) { STransAction *pAction = taosArrayGet(pTrans->undoActions, i); - SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet), TRANS_ENCODE_OVER) - SDB_SET_INT16(pRaw, dataPos, pAction->msgType, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->contLen, TRANS_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, (void *)pAction->pCont, pAction->contLen, TRANS_ENCODE_OVER) + SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet), _OVER) + SDB_SET_INT16(pRaw, dataPos, pAction->msgType, _OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->contLen, _OVER) + SDB_SET_BINARY(pRaw, dataPos, (void *)pAction->pCont, pAction->contLen, _OVER) } - SDB_SET_INT32(pRaw, dataPos, pTrans->startFunc, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pTrans->stopFunc, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pTrans->paramLen, TRANS_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pTrans->startFunc, _OVER) + SDB_SET_INT32(pRaw, dataPos, pTrans->stopFunc, _OVER) + SDB_SET_INT32(pRaw, dataPos, pTrans->paramLen, _OVER) if (pTrans->param != NULL) { - SDB_SET_BINARY(pRaw, dataPos, pTrans->param, pTrans->paramLen, TRANS_ENCODE_OVER) + SDB_SET_BINARY(pRaw, dataPos, pTrans->param, pTrans->paramLen, _OVER) } - SDB_SET_RESERVE(pRaw, dataPos, TRANS_RESERVE_SIZE, TRANS_ENCODE_OVER) - SDB_SET_DATALEN(pRaw, dataPos, TRANS_ENCODE_OVER) + SDB_SET_RESERVE(pRaw, dataPos, TRANS_RESERVE_SIZE, _OVER) + SDB_SET_DATALEN(pRaw, dataPos, _OVER) terrno = 0; -TRANS_ENCODE_OVER: +_OVER: if (terrno != 0) { mError("trans:%d, failed to encode to raw:%p len:%d since %s", pTrans->id, pRaw, dataPos, terrstr()); sdbFreeRaw(pRaw); @@ -229,14 +240,14 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &pTrans->id, _OVER) - int16_t type = 0; - int16_t policy = 0; int16_t stage = 0; - SDB_GET_INT16(pRaw, dataPos, &policy, _OVER) + int16_t policy = 0; + int16_t type = 0; SDB_GET_INT16(pRaw, dataPos, &stage, _OVER) + SDB_GET_INT16(pRaw, dataPos, &policy, _OVER) SDB_GET_INT16(pRaw, dataPos, &type, _OVER) - pTrans->policy = policy; pTrans->stage = stage; + pTrans->policy = policy; pTrans->type = type; SDB_GET_INT64(pRaw, dataPos, &pTrans->createdTime, _OVER) SDB_GET_INT64(pRaw, dataPos, &pTrans->dbUid, _OVER) @@ -453,7 +464,6 @@ static TransCbFp mndTransGetCbFp(ETrnFuncType ftype) { } static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) { - // pTrans->stage = TRN_STAGE_PREPARE; mTrace("trans:%d, perform insert action, row:%p stage:%s", pTrans->id, pTrans, mndTransStr(pTrans->stage)); if (pTrans->startFunc > 0) { @@ -516,8 +526,7 @@ static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOld, STrans *pNew) { } static STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId) { - SSdb *pSdb = pMnode->pSdb; - STrans *pTrans = sdbAcquire(pSdb, SDB_TRANS, &transId); + STrans *pTrans = sdbAcquire(pMnode->pSdb, SDB_TRANS, &transId); if (pTrans == NULL) { terrno = TSDB_CODE_MND_TRANS_NOT_EXIST; } @@ -558,7 +567,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnType type, const S return NULL; } - mDebug("trans:%d, local var is created, data:%p", pTrans->id, pTrans); + mDebug("trans:%d, local object is created, data:%p", pTrans->id, pTrans); return pTrans; } @@ -585,7 +594,7 @@ static void mndTransDropActions(SArray *pArray) { void mndTransDrop(STrans *pTrans) { if (pTrans != NULL) { mndTransDropData(pTrans); - mDebug("trans:%d, local var is freed, data:%p", pTrans->id, pTrans); + mDebug("trans:%d, local object is freed, data:%p", pTrans->id, pTrans); taosMemoryFreeClear(pTrans); } } @@ -674,19 +683,19 @@ static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) { } static bool mndIsBasicTrans(STrans *pTrans) { - return pTrans->stage > TRN_TYPE_BASIC_SCOPE && pTrans->stage < TRN_TYPE_BASIC_SCOPE_END; + return pTrans->type > TRN_TYPE_BASIC_SCOPE && pTrans->type < TRN_TYPE_BASIC_SCOPE_END; } static bool mndIsGlobalTrans(STrans *pTrans) { - return pTrans->stage > TRN_TYPE_GLOBAL_SCOPE && pTrans->stage < TRN_TYPE_GLOBAL_SCOPE_END; + return pTrans->type > TRN_TYPE_GLOBAL_SCOPE && pTrans->type < TRN_TYPE_GLOBAL_SCOPE_END; } static bool mndIsDbTrans(STrans *pTrans) { - return pTrans->stage > TRN_TYPE_DB_SCOPE && pTrans->stage < TRN_TYPE_DB_SCOPE_END; + return pTrans->type > TRN_TYPE_DB_SCOPE && pTrans->type < TRN_TYPE_DB_SCOPE_END; } static bool mndIsStbTrans(STrans *pTrans) { - return pTrans->stage > TRN_TYPE_STB_SCOPE && pTrans->stage < TRN_TYPE_STB_SCOPE_END; + return pTrans->type > TRN_TYPE_STB_SCOPE && pTrans->type < TRN_TYPE_STB_SCOPE_END; } static bool mndCheckTransCanParallel(SMnode *pMnode, STrans *pNewTrans) { From 7eede87869b466e804645d565ea1f42fa27891d3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 4 May 2022 16:16:35 +0800 Subject: [PATCH 32/48] fix: reset confict trans types --- source/dnode/mnode/impl/inc/mndDef.h | 1 + source/dnode/mnode/impl/src/mndTrans.c | 61 +++++++++++-------- source/dnode/mnode/impl/test/trans/trans2.cpp | 2 +- 3 files changed, 36 insertions(+), 28 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index a3c143df41..d516b0bf26 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -72,6 +72,7 @@ typedef enum { TRN_TYPE_DROP_USER = 1003, TRN_TYPE_CREATE_FUNC = 1004, TRN_TYPE_DROP_FUNC = 1005, + TRN_TYPE_CREATE_SNODE = 1006, TRN_TYPE_DROP_SNODE = 1007, TRN_TYPE_CREATE_QNODE = 1008, diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index f3d7bf8c66..ce2a0efd2f 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -63,13 +63,15 @@ static int32_t mndRetrieveTrans(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pB static void mndCancelGetNextTrans(SMnode *pMnode, void *pIter); int32_t mndInitTrans(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_TRANS, - .keyType = SDB_KEY_INT32, - .encodeFp = (SdbEncodeFp)mndTransActionEncode, - .decodeFp = (SdbDecodeFp)mndTransActionDecode, - .insertFp = (SdbInsertFp)mndTransActionInsert, - .updateFp = (SdbUpdateFp)mndTransActionUpdate, - .deleteFp = (SdbDeleteFp)mndTransActionDelete}; + SSdbTable table = { + .sdbType = SDB_TRANS, + .keyType = SDB_KEY_INT32, + .encodeFp = (SdbEncodeFp)mndTransActionEncode, + .decodeFp = (SdbDecodeFp)mndTransActionDecode, + .insertFp = (SdbInsertFp)mndTransActionInsert, + .updateFp = (SdbUpdateFp)mndTransActionUpdate, + .deleteFp = (SdbDeleteFp)mndTransActionDelete, + }; mndSetMsgHandle(pMnode, TDMT_MND_TRANS_TIMER, mndProcessTransReq); mndSetMsgHandle(pMnode, TDMT_MND_KILL_TRANS, mndProcessKillTransReq); @@ -411,6 +413,16 @@ static const char *mndTransType(ETrnType type) { return "subscribe"; case TRN_TYPE_REBALANCE: return "rebalance"; + case TRN_TYPE_COMMIT_OFFSET: + return "commit-offset"; + case TRN_TYPE_CREATE_STREAM: + return "create-stream"; + case TRN_TYPE_DROP_STREAM: + return "drop-stream"; + case TRN_TYPE_CONSUMER_LOST: + return "consumer-lost"; + case TRN_TYPE_CONSUMER_RECOVER: + return "consumer-recover"; case TRN_TYPE_CREATE_DNODE: return "create-qnode"; case TRN_TYPE_DROP_DNODE: @@ -713,41 +725,34 @@ static bool mndCheckTransCanParallel(SMnode *pMnode, STrans *pNewTrans) { if (mndIsDbTrans(pTrans) || mndIsStbTrans(pTrans)) { mError("trans:%d, can't execute since trans:%d in progress db:%s", pNewTrans->id, pTrans->id, pTrans->dbname); canParallel = false; - break; + } else { } } - if (mndIsDbTrans(pNewTrans)) { - if (mndIsBasicTrans(pTrans)) continue; + else if (mndIsDbTrans(pNewTrans)) { if (mndIsGlobalTrans(pTrans)) { mError("trans:%d, can't execute since trans:%d in progress", pNewTrans->id, pTrans->id); canParallel = false; - break; - } - if (mndIsDbTrans(pTrans) || mndIsStbTrans(pTrans)) { + } else if (mndIsDbTrans(pTrans) || mndIsStbTrans(pTrans)) { if (pNewTrans->dbUid == pTrans->dbUid) { mError("trans:%d, can't execute since trans:%d in progress db:%s", pNewTrans->id, pTrans->id, pTrans->dbname); canParallel = false; - break; } + } else { } } - if (mndIsStbTrans(pNewTrans)) { - if (mndIsBasicTrans(pTrans)) continue; + else if (mndIsStbTrans(pNewTrans)) { if (mndIsGlobalTrans(pTrans)) { mError("trans:%d, can't execute since trans:%d in progress", pNewTrans->id, pTrans->id); canParallel = false; - break; - } - if (mndIsDbTrans(pTrans)) { + } else if (mndIsDbTrans(pTrans)) { if (pNewTrans->dbUid == pTrans->dbUid) { mError("trans:%d, can't execute since trans:%d in progress db:%s", pNewTrans->id, pTrans->id, pTrans->dbname); canParallel = false; - break; } + } else { } - if (mndIsStbTrans(pTrans)) continue; } sdbRelease(pMnode->pSdb, pTrans); @@ -842,12 +847,14 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { mDebug("trans:%d, send rsp, code:0x%04x stage:%d app:%p", pTrans->id, pTrans->code & 0xFFFF, pTrans->stage, pTrans->rpcAHandle); - SRpcMsg rspMsg = {.handle = pTrans->rpcHandle, - .code = pTrans->code, - .ahandle = pTrans->rpcAHandle, - .refId = pTrans->rpcRefId, - .pCont = rpcCont, - .contLen = pTrans->rpcRspLen}; + SRpcMsg rspMsg = { + .handle = pTrans->rpcHandle, + .ahandle = pTrans->rpcAHandle, + .refId = pTrans->rpcRefId, + .code = pTrans->code, + .pCont = rpcCont, + .contLen = pTrans->rpcRspLen, + }; tmsgSendRsp(&rspMsg); pTrans->rpcHandle = NULL; pTrans->rpcRsp = NULL; diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp index abab24723f..252cd105a0 100644 --- a/source/dnode/mnode/impl/test/trans/trans2.cpp +++ b/source/dnode/mnode/impl/test/trans/trans2.cpp @@ -22,7 +22,7 @@ class MndTestTrans2 : public ::testing::Test { static void SetUpTestSuite() { dDebugFlag = 143; vDebugFlag = 0; - mDebugFlag = 143; + mDebugFlag = 207; cDebugFlag = 0; jniDebugFlag = 0; tmrDebugFlag = 135; From 5b2b4da11bf0c9cd5604a456b35ec1d83b7275a8 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 4 May 2022 16:49:24 +0800 Subject: [PATCH 33/48] refactor: fix: reset confict trans types --- include/util/taoserror.h | 3 ++- source/dnode/mnode/impl/src/mndTrans.c | 31 ++++++++++++++------------ source/util/src/terror.c | 3 ++- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index b8baf99552..abc752955d 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -264,7 +264,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_TRANS_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03D0) #define TSDB_CODE_MND_TRANS_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03D1) #define TSDB_CODE_MND_TRANS_INVALID_STAGE TAOS_DEF_ERROR_CODE(0, 0x03D2) -#define TSDB_CODE_MND_TRANS_CAN_NOT_PARALLEL TAOS_DEF_ERROR_CODE(0, 0x03D4) +#define TSDB_CODE_MND_TRANS_CONFLICT TAOS_DEF_ERROR_CODE(0, 0x03D3) +#define TSDB_CODE_MND_TRANS_UNKNOW_ERROR TAOS_DEF_ERROR_CODE(0, 0x03D4) // mnode-mq #define TSDB_CODE_MND_TOPIC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E0) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index ce2a0efd2f..1c238d4082 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -710,12 +710,12 @@ static bool mndIsStbTrans(STrans *pTrans) { return pTrans->type > TRN_TYPE_STB_SCOPE && pTrans->type < TRN_TYPE_STB_SCOPE_END; } -static bool mndCheckTransCanParallel(SMnode *pMnode, STrans *pNewTrans) { +static bool mndCheckTransConflict(SMnode *pMnode, STrans *pNewTrans) { STrans *pTrans = NULL; void *pIter = NULL; - bool canParallel = true; + bool conflict = false; - if (mndIsBasicTrans(pNewTrans)) return canParallel; + if (mndIsBasicTrans(pNewTrans)) return conflict; while (1) { pIter = sdbFetch(pMnode->pSdb, SDB_TRANS, pIter, (void **)&pTrans); @@ -724,7 +724,7 @@ static bool mndCheckTransCanParallel(SMnode *pMnode, STrans *pNewTrans) { if (mndIsGlobalTrans(pNewTrans)) { if (mndIsDbTrans(pTrans) || mndIsStbTrans(pTrans)) { mError("trans:%d, can't execute since trans:%d in progress db:%s", pNewTrans->id, pTrans->id, pTrans->dbname); - canParallel = false; + conflict = true; } else { } } @@ -732,11 +732,11 @@ static bool mndCheckTransCanParallel(SMnode *pMnode, STrans *pNewTrans) { else if (mndIsDbTrans(pNewTrans)) { if (mndIsGlobalTrans(pTrans)) { mError("trans:%d, can't execute since trans:%d in progress", pNewTrans->id, pTrans->id); - canParallel = false; + conflict = true; } else if (mndIsDbTrans(pTrans) || mndIsStbTrans(pTrans)) { if (pNewTrans->dbUid == pTrans->dbUid) { mError("trans:%d, can't execute since trans:%d in progress db:%s", pNewTrans->id, pTrans->id, pTrans->dbname); - canParallel = false; + conflict = true; } } else { } @@ -745,11 +745,11 @@ static bool mndCheckTransCanParallel(SMnode *pMnode, STrans *pNewTrans) { else if (mndIsStbTrans(pNewTrans)) { if (mndIsGlobalTrans(pTrans)) { mError("trans:%d, can't execute since trans:%d in progress", pNewTrans->id, pTrans->id); - canParallel = false; + conflict = true; } else if (mndIsDbTrans(pTrans)) { if (pNewTrans->dbUid == pTrans->dbUid) { mError("trans:%d, can't execute since trans:%d in progress db:%s", pNewTrans->id, pTrans->id, pTrans->dbname); - canParallel = false; + conflict = true; } } else { } @@ -760,12 +760,12 @@ static bool mndCheckTransCanParallel(SMnode *pMnode, STrans *pNewTrans) { sdbCancelFetch(pMnode->pSdb, pIter); sdbRelease(pMnode->pSdb, pTrans); - return canParallel; + return conflict; } int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { - if (!mndCheckTransCanParallel(pMnode, pTrans)) { - terrno = TSDB_CODE_MND_TRANS_CAN_NOT_PARALLEL; + if (mndCheckTransConflict(pMnode, pTrans)) { + terrno = TSDB_CODE_MND_TRANS_CONFLICT; mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); return -1; } @@ -819,7 +819,8 @@ static int32_t mndTransRollback(SMnode *pMnode, STrans *pTrans) { } static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { - bool sendRsp = false; + bool sendRsp = false; + int32_t code = pTrans->code; if (pTrans->stage == TRN_STAGE_FINISHED) { sendRsp = true; @@ -829,12 +830,14 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { if (pTrans->stage == TRN_STAGE_UNDO_LOG || pTrans->stage == TRN_STAGE_UNDO_ACTION || pTrans->stage == TRN_STAGE_ROLLBACK) { sendRsp = true; + if (code == 0) code = TSDB_CODE_MND_TRANS_UNKNOW_ERROR; } } if (pTrans->policy == TRN_POLICY_RETRY) { if (pTrans->stage == TRN_STAGE_REDO_ACTION && pTrans->failedTimes > 0) { sendRsp = true; + if (code == 0) code = TSDB_CODE_MND_TRANS_UNKNOW_ERROR; } } @@ -845,13 +848,13 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { } taosMemoryFree(pTrans->rpcRsp); - mDebug("trans:%d, send rsp, code:0x%04x stage:%d app:%p", pTrans->id, pTrans->code & 0xFFFF, pTrans->stage, + mDebug("trans:%d, send rsp, code:0x%04x stage:%d app:%p", pTrans->id, code & 0xFFFF, pTrans->stage, pTrans->rpcAHandle); SRpcMsg rspMsg = { .handle = pTrans->rpcHandle, .ahandle = pTrans->rpcAHandle, .refId = pTrans->rpcRefId, - .code = pTrans->code, + .code = code, .pCont = rpcCont, .contLen = pTrans->rpcRspLen, }; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index bd787d50b0..00fe8bd0e9 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -271,7 +271,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_RETRIEVE, "Invalid func retrieve TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_ALREADY_EXIST, "Transaction already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_NOT_EXIST, "Transaction not exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_INVALID_STAGE, "Invalid stage to kill") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CAN_NOT_PARALLEL, "Conflicting transaction not completed") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CONFLICT, "Conflict transaction not completed") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_UNKNOW_ERROR, "Unknown transaction error") // mnode-mq TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_ALREADY_EXIST, "Topic already exists") From 71021334fb6a928f38a0e926010e6ce51c7388b4 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 4 May 2022 20:37:37 +0800 Subject: [PATCH 34/48] refactor: adjust transaction code --- source/dnode/mnode/impl/src/mndTrans.c | 12 +++--- source/dnode/mnode/impl/src/mndUser.c | 3 -- source/dnode/mnode/impl/test/trans/trans2.cpp | 39 ++++++++++++++----- 3 files changed, 34 insertions(+), 20 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 1c238d4082..e80a16b16e 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -829,15 +829,13 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { if (pTrans->policy == TRN_POLICY_ROLLBACK) { if (pTrans->stage == TRN_STAGE_UNDO_LOG || pTrans->stage == TRN_STAGE_UNDO_ACTION || pTrans->stage == TRN_STAGE_ROLLBACK) { - sendRsp = true; if (code == 0) code = TSDB_CODE_MND_TRANS_UNKNOW_ERROR; + sendRsp = true; } - } - - if (pTrans->policy == TRN_POLICY_RETRY) { + } else { if (pTrans->stage == TRN_STAGE_REDO_ACTION && pTrans->failedTimes > 0) { - sendRsp = true; if (code == 0) code = TSDB_CODE_MND_TRANS_UNKNOW_ERROR; + sendRsp = true; } } @@ -1102,8 +1100,8 @@ static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans) { } else { pTrans->code = terrno; if (pTrans->policy == TRN_POLICY_ROLLBACK) { - pTrans->stage = TRN_STAGE_REDO_ACTION; - mError("trans:%d, stage from commit to redoAction since %s, failedTimes:%d", pTrans->id, terrstr(), + pTrans->stage = TRN_STAGE_UNDO_ACTION; + mError("trans:%d, stage from commit to undoAction since %s, failedTimes:%d", pTrans->id, terrstr(), pTrans->failedTimes); continueExec = true; } else { diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 1800d24cf2..5e15bdeb43 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -275,9 +275,6 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate } sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); - char *param = strdup("====> test code to be deleted later <====="); - mndTransSetCb(pTrans, TEST_TRANS_START_FUNC, TEST_TRANS_STOP_FUNC, param, strlen(param) + 1); - if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp index 252cd105a0..aaf2c649e4 100644 --- a/source/dnode/mnode/impl/test/trans/trans2.cpp +++ b/source/dnode/mnode/impl/test/trans/trans2.cpp @@ -19,7 +19,7 @@ void reportStartup(SMgmtWrapper *pWrapper, const char *name, const char *desc) { class MndTestTrans2 : public ::testing::Test { protected: - static void SetUpTestSuite() { + static void InitLog() { dDebugFlag = 143; vDebugFlag = 0; mDebugFlag = 207; @@ -42,9 +42,9 @@ class MndTestTrans2 : public ::testing::Test { if (taosInitLog("taosdlog", 1) != 0) { printf("failed to init log file\n"); } + } - walInit(); - + static void InitMnode() { static SMsgCb msgCb = {0}; msgCb.reportStartupFp = reportStartup; msgCb.pWrapper = (SMgmtWrapper *)(&msgCb); // hack @@ -63,6 +63,12 @@ class MndTestTrans2 : public ::testing::Test { pMnode = mndOpen(mnodepath, &opt); } + static void SetUpTestSuite() { + InitLog(); + walInit(); + InitMnode(); + } + static void TearDownTestSuite() { mndClose(pMnode); walCleanUp(); @@ -76,11 +82,11 @@ class MndTestTrans2 : public ::testing::Test { void SetUp() override {} void TearDown() override {} - void CreateUser(const char *user) { + int32_t CreateUser(const char *acct, const char *user) { SUserObj userObj = {0}; taosEncryptPass_c((uint8_t *)"taosdata", strlen("taosdata"), userObj.pass); tstrncpy(userObj.user, user, TSDB_USER_LEN); - tstrncpy(userObj.acct, "root", TSDB_USER_LEN); + tstrncpy(userObj.acct, acct, TSDB_USER_LEN); userObj.createdTime = taosGetTimestampMs(); userObj.updateTime = userObj.createdTime; userObj.superUser = 1; @@ -94,19 +100,32 @@ class MndTestTrans2 : public ::testing::Test { char *param = strdup("====> test param <====="); mndTransSetCb(pTrans, TEST_TRANS_START_FUNC, TEST_TRANS_STOP_FUNC, param, strlen(param) + 1); - mndTransPrepare(pMnode, pTrans); + int32_t code = mndTransPrepare(pMnode, pTrans); mndTransDrop(pTrans); + + return code; } }; SMnode *MndTestTrans2::pMnode; TEST_F(MndTestTrans2, 01_CbFunc) { + const char *acct = "root"; + const char *acct_invalid = "root1"; + const char *user1 = "test1"; + const char *user2 = "test2"; + SUserObj *pUser1 = NULL; + SUserObj *pUser2 = NULL; + ASSERT_NE(pMnode, nullptr); - const char *user1 = "test1"; - CreateUser(user1); - - SUserObj *pUser1 = mndAcquireUser(pMnode, user1); + // create user success + EXPECT_EQ(CreateUser(acct, user1), 0); + pUser1 = mndAcquireUser(pMnode, user1); ASSERT_NE(pUser1, nullptr); + + // failed to create user and rollback + EXPECT_EQ(CreateUser(acct_invalid, user2), 0); + pUser2 = mndAcquireUser(pMnode, user2); + ASSERT_EQ(pUser2, nullptr); } From fb7da7e566bedc0af30758785c29c30a7a4b2999 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 4 May 2022 21:02:27 +0800 Subject: [PATCH 35/48] enh(index): support index filter --- source/libs/index/inc/indexCache.h | 1 - source/libs/index/inc/indexFst.h | 7 ++-- source/libs/index/inc/indexInt.h | 1 + source/libs/index/src/indexCache.c | 60 ++++++++++++++++++++++++++---- source/libs/index/src/indexTfile.c | 43 +++++++++++++++++---- 5 files changed, 91 insertions(+), 21 deletions(-) diff --git a/source/libs/index/inc/indexCache.h b/source/libs/index/inc/indexCache.h index 44355db7a7..3ea986ad48 100644 --- a/source/libs/index/inc/indexCache.h +++ b/source/libs/index/inc/indexCache.h @@ -56,7 +56,6 @@ typedef struct CacheTerm { int8_t colType; SIndexOperOnColumn operaType; - int8_t qType; // query type } CacheTerm; // diff --git a/source/libs/index/inc/indexFst.h b/source/libs/index/inc/indexFst.h index 39ad5ffa8c..0a360c1c72 100644 --- a/source/libs/index/inc/indexFst.h +++ b/source/libs/index/inc/indexFst.h @@ -52,7 +52,6 @@ typedef struct FstRange { uint64_t end; } FstRange; -typedef enum { GE, GT, LE, LT } RangeType; typedef enum { OneTransNext, OneTrans, AnyTrans, EmptyFinal } State; typedef enum { Ordered, OutOfOrdered, DuplicateKey } OrderType; @@ -174,9 +173,9 @@ Output fstStateFinalOutput(FstState* state, uint64_t version, FstSlice* date, uint64_t fstStateFindInput(FstState* state, FstNode* node, uint8_t b, bool* null); #define FST_STATE_ONE_TRNAS_NEXT(node) (node->state.state == OneTransNext) -#define FST_STATE_ONE_TRNAS(node) (node->state.state == OneTrans) -#define FST_STATE_ANY_TRANS(node) (node->state.state == AnyTrans) -#define FST_STATE_EMPTY_FINAL(node) (node->state.state == EmptyFinal) +#define FST_STATE_ONE_TRNAS(node) (node->state.state == OneTrans) +#define FST_STATE_ANY_TRANS(node) (node->state.state == AnyTrans) +#define FST_STATE_EMPTY_FINAL(node) (node->state.state == EmptyFinal) typedef struct FstLastTransition { uint8_t inp; diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 04817c4df3..5c7b8b9afe 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -34,6 +34,7 @@ extern "C" { #endif +typedef enum { LT, LE, GT, GE } RangeType; typedef enum { kTypeValue, kTypeDeletion } STermValueType; typedef struct SIndexStat { diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index ea3cbf5658..13768ce682 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -43,6 +43,20 @@ static int32_t cacheSearchLessEqual(void* cache, CacheTerm* ct, SIdxTempResult* static int32_t cacheSearchGreaterThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); static int32_t cacheSearchGreaterEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); static int32_t cacheSearchRange(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); +/*comm func of compare, used in (LE/LT/GE/GT compare)*/ +static int32_t cacheSearchCompareFunc(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s, + RangeType type); + +typedef enum { MATCH, CONTINUE, BREAK } TExeCond; +typedef TExeCond (*_cache_range_compare)(void* a, void* b, int8_t type); + +static TExeCond tCompareLessThan(void* a, void* b, int8_t type) { return MATCH; } +static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) { return MATCH; } +static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) { return MATCH; } +static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { return MATCH; } + +static TExeCond (*rangeCompare[])(void* a, void* b, int8_t type) = {tCompareLessThan, tCompareLessEqual, + tCompareGreaterThan, tCompareGreaterEqual}; static int32_t (*cacheSearch[])(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) = { cacheSearchTerm, cacheSearchPrefix, cacheSearchSuffix, cacheSearchRegex, cacheSearchLessThan, @@ -93,21 +107,51 @@ static int32_t cacheSearchRegex(void* cache, CacheTerm* ct, SIdxTempResult* tr, // impl later return 0; } +static int32_t cacheSearchCompareFunc(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s, + RangeType type) { + if (cache == NULL) { + return 0; + } + _cache_range_compare cmpFn = rangeCompare[type]; + + MemTable* mem = cache; + char* key = indexCacheTermGet(ct); + + SSkipListIterator* iter = tSkipListCreateIter(mem->mem); + while (tSkipListIterNext(iter)) { + SSkipListNode* node = tSkipListIterGet(iter); + if (node == NULL) { + break; + } + CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node); + TExeCond cond = cmpFn(c->colVal, ct->colVal, ct->colType); + if (cond == MATCH) { + if (c->operaType == ADD_VALUE) { + INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid) + // taosArrayPush(result, &c->uid); + *s = kTypeValue; + } else if (c->operaType == DEL_VALUE) { + INDEX_MERGE_ADD_DEL(tr->added, tr->deled, c->uid) + } + } else if (cond == CONTINUE) { + } else if (cond == BREAK) { + break; + } + } + tSkipListDestroyIter(iter); + return TSDB_CODE_SUCCESS; +} static int32_t cacheSearchLessThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { - // impl later - return 0; + return cacheSearchCompareFunc(cache, ct, tr, s, LT); } static int32_t cacheSearchLessEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { - // impl later - return 0; + return cacheSearchCompareFunc(cache, ct, tr, s, LE); } static int32_t cacheSearchGreaterThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { - // impl later - return 0; + return cacheSearchCompareFunc(cache, ct, tr, s, GT); } static int32_t cacheSearchGreaterEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { - // impl later - return 0; + return cacheSearchCompareFunc(cache, ct, tr, s, GE); } static int32_t cacheSearchRange(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { // impl later diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 327a06084c..5aed2bd6b0 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -70,6 +70,8 @@ static int32_t tfSearchGreaterThan(void* reader, SIndexTerm* tem, SIdxTempResult static int32_t tfSearchGreaterEqual(void* reader, SIndexTerm* tem, SIdxTempResult* tr); static int32_t tfSearchRange(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTempResult* tr, RangeType ctype); + static int32_t (*tfSearch[])(void* reader, SIndexTerm* tem, SIdxTempResult* tr) = { tfSearchTerm, tfSearchPrefix, tfSearchSuffix, tfSearchRegex, tfSearchLessThan, tfSearchLessEqual, tfSearchGreaterThan, tfSearchGreaterEqual, tfSearchRange}; @@ -304,21 +306,46 @@ static int32_t tfSearchRegex(void* reader, SIndexTerm* tem, SIdxTempResult* tr) fstSliceDestroy(&key); return 0; } + +static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTempResult* tr, RangeType type) { + bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON); + int ret = 0; + char* p = tem->colVal; + uint64_t sz = tem->nColVal; + if (hasJson) { + p = indexPackJsonData(tem); + sz = strlen(p); + } + SArray* offsets = taosArrayInit(16, sizeof(uint64_t)); + + AutomationCtx* ctx = automCtxCreate((void*)p, AUTOMATION_ALWAYS); + FstStreamBuilder* sb = fstSearch(((TFileReader*)reader)->fst, ctx); + + FstSlice h = fstSliceCreate((uint8_t*)p, sz); + fstStreamBuilderSetRange(sb, &h, type); + fstSliceDestroy(&h); + + StreamWithState* st = streamBuilderIntoStream(sb); + StreamWithStateResult* rt = NULL; + while ((rt = streamWithStateNextWith(st, NULL)) != NULL) { + taosArrayPush(offsets, &(rt->out.out)); + swsResultDestroy(rt); + } + streamWithStateDestroy(st); + fstStreamBuilderDestroy(sb); + return TSDB_CODE_SUCCESS; +} static int32_t tfSearchLessThan(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { - // impl later - return 0; + return tfSearchCompareFunc(reader, tem, tr, LT); } static int32_t tfSearchLessEqual(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { - // impl later - return 0; + return tfSearchCompareFunc(reader, tem, tr, LE); } static int32_t tfSearchGreaterThan(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { - // impl later - return 0; + return tfSearchCompareFunc(reader, tem, tr, GT); } static int32_t tfSearchGreaterEqual(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { - // impl later - return 0; + return tfSearchCompareFunc(reader, tem, tr, GE); } static int32_t tfSearchRange(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON); From 2f42c2e7933f2231c15f2236fe17b86d70c49fca Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 4 May 2022 22:00:04 +0800 Subject: [PATCH 36/48] refactor: adjust transaction code --- include/common/tglobal.h | 4 ++ source/common/src/tglobal.c | 4 ++ source/common/src/tmsgcb.c | 62 ++++++++++++++++--- source/dnode/mnode/impl/src/mndTrans.c | 6 +- source/dnode/mnode/impl/src/mnode.c | 11 ++-- source/dnode/mnode/impl/test/trans/trans2.cpp | 6 ++ 6 files changed, 75 insertions(+), 18 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index fc3d575317..f253d31963 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -121,6 +121,10 @@ extern char tsCompressor[]; extern int32_t tsDiskCfgNum; extern SDiskCfg tsDiskCfg[]; +// internal +extern int32_t tsTransPullupMs; +extern int32_t tsMaRebalanceMs; + #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, const char *envFile, diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index bbf14b2fdc..4470f5b0a3 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -169,6 +169,10 @@ uint32_t tsMaxRange = 500; // max range uint32_t tsCurRange = 100; // range char tsCompressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR +// internal +int32_t tsTransPullupMs = 6000; +int32_t tsMaRebalanceMs = 2000; + void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary) { tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN); tsDiskCfg[index].level = level; diff --git a/source/common/src/tmsgcb.c b/source/common/src/tmsgcb.c index c9cbb73884..42612cecb9 100644 --- a/source/common/src/tmsgcb.c +++ b/source/common/src/tmsgcb.c @@ -15,37 +15,83 @@ #define _DEFAULT_SOURCE #include "tmsgcb.h" +#include "taoserror.h" static SMsgCb tsDefaultMsgCb; void tmsgSetDefaultMsgCb(const SMsgCb* pMsgCb) { tsDefaultMsgCb = *pMsgCb; } int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pReq) { - return (*pMsgCb->queueFps[qtype])(pMsgCb->pWrapper, pReq); + PutToQueueFp fp = pMsgCb->queueFps[qtype]; + if (fp != NULL) { + return (*fp)(pMsgCb->pWrapper, pReq); + } else { + terrno = TSDB_CODE_INVALID_PTR; + return -1; + } } int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype) { - return (*pMsgCb->qsizeFp)(pMsgCb->pWrapper, vgId, qtype); + GetQueueSizeFp fp = pMsgCb->qsizeFp; + if (fp != NULL) { + return (*fp)(pMsgCb->pWrapper, vgId, qtype); + } else { + terrno = TSDB_CODE_INVALID_PTR; + return -1; + } } int32_t tmsgSendReq(const SMsgCb* pMsgCb, const SEpSet* epSet, SRpcMsg* pReq) { - return (*pMsgCb->sendReqFp)(pMsgCb->pWrapper, epSet, pReq); + SendReqFp fp = pMsgCb->sendReqFp; + if (fp != NULL) { + return (*fp)(pMsgCb->pWrapper, epSet, pReq); + } else { + terrno = TSDB_CODE_INVALID_PTR; + return -1; + } } -void tmsgSendRsp(const SRpcMsg* pRsp) { return (*tsDefaultMsgCb.sendRspFp)(tsDefaultMsgCb.pWrapper, pRsp); } +void tmsgSendRsp(const SRpcMsg* pRsp) { + SendRspFp fp = tsDefaultMsgCb.sendRspFp; + if (fp != NULL) { + return (*fp)(tsDefaultMsgCb.pWrapper, pRsp); + } else { + terrno = TSDB_CODE_INVALID_PTR; + } +} void tmsgSendRedirectRsp(const SRpcMsg* pRsp, const SEpSet* pNewEpSet) { - return (*tsDefaultMsgCb.sendRedirectRspFp)(tsDefaultMsgCb.pWrapper, pRsp, pNewEpSet); + SendRedirectRspFp fp = tsDefaultMsgCb.sendRedirectRspFp; + if (fp != NULL) { + (*fp)(tsDefaultMsgCb.pWrapper, pRsp, pNewEpSet); + } else { + terrno = TSDB_CODE_INVALID_PTR; + } } void tmsgRegisterBrokenLinkArg(const SMsgCb* pMsgCb, SRpcMsg* pMsg) { - (*pMsgCb->registerBrokenLinkArgFp)(pMsgCb->pWrapper, pMsg); + RegisterBrokenLinkArgFp fp = pMsgCb->registerBrokenLinkArgFp; + if (fp != NULL) { + (*fp)(pMsgCb->pWrapper, pMsg); + } else { + terrno = TSDB_CODE_INVALID_PTR; + } } void tmsgReleaseHandle(void* handle, int8_t type) { - (*tsDefaultMsgCb.releaseHandleFp)(tsDefaultMsgCb.pWrapper, handle, type); + ReleaseHandleFp fp = tsDefaultMsgCb.releaseHandleFp; + if (fp != NULL) { + (*fp)(tsDefaultMsgCb.pWrapper, handle, type); + } else { + terrno = TSDB_CODE_INVALID_PTR; + } } void tmsgReportStartup(const char* name, const char* desc) { - (*tsDefaultMsgCb.reportStartupFp)(tsDefaultMsgCb.pWrapper, name, desc); + ReportStartup fp = tsDefaultMsgCb.reportStartupFp; + if (fp != NULL && tsDefaultMsgCb.pWrapper != NULL) { + (*fp)(tsDefaultMsgCb.pWrapper, name, desc); + } else { + terrno = TSDB_CODE_INVALID_PTR; + } } \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index e80a16b16e..6110d0be44 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1126,7 +1126,7 @@ static bool mndTransPerformCommitLogStage(SMnode *pMnode, STrans *pTrans) { } else { pTrans->code = terrno; pTrans->failedTimes++; - mError("trans:%d, stage keep on commitLog since %s", pTrans->id, terrstr()); + mError("trans:%d, stage keep on commitLog since %s, failedTimes:%d", pTrans->id, terrstr(), pTrans->failedTimes); continueExec = false; } @@ -1162,7 +1162,7 @@ static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans) { continueExec = false; } else { pTrans->failedTimes++; - mError("trans:%d, stage keep on undoAction since %s", pTrans->id, terrstr()); + mError("trans:%d, stage keep on undoAction since %s, failedTimes:%d", pTrans->id, terrstr(), pTrans->failedTimes); continueExec = false; } @@ -1179,7 +1179,7 @@ static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans) { continueExec = true; } else { pTrans->failedTimes++; - mError("trans:%d, stage keep on rollback since %s", pTrans->id, terrstr()); + mError("trans:%d, stage keep on rollback since %s, failedTimes:%d", pTrans->id, terrstr(), pTrans->failedTimes); continueExec = false; } diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index b38fd6178c..e2814e95f0 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -43,9 +43,6 @@ #include "mndUser.h" #include "mndVgroup.h" -#define MQ_TIMER_MS 2000 -#define TRNAS_TIMER_MS 6000 - static void *mndBuildTimerMsg(int32_t *pContLen) { SMTimerReq timerReq = {0}; @@ -68,7 +65,7 @@ static void mndPullupTrans(void *param, void *tmrId) { tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); } - taosTmrReset(mndPullupTrans, TRNAS_TIMER_MS, pMnode, pMnode->timer, &pMnode->transTimer); + taosTmrReset(mndPullupTrans, tsTransPullupMs, pMnode, pMnode->timer, &pMnode->transTimer); } static void mndCalMqRebalance(void *param, void *tmrId) { @@ -84,7 +81,7 @@ static void mndCalMqRebalance(void *param, void *tmrId) { tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); } - taosTmrReset(mndCalMqRebalance, MQ_TIMER_MS, pMnode, pMnode->timer, &pMnode->mqTimer); + taosTmrReset(mndCalMqRebalance, tsMaRebalanceMs, pMnode, pMnode->timer, &pMnode->mqTimer); } static void mndPullupTelem(void *param, void *tmrId) { @@ -106,12 +103,12 @@ static int32_t mndInitTimer(SMnode *pMnode) { return -1; } - if (taosTmrReset(mndPullupTrans, TRNAS_TIMER_MS, pMnode, pMnode->timer, &pMnode->transTimer)) { + if (taosTmrReset(mndPullupTrans, tsTransPullupMs, pMnode, pMnode->timer, &pMnode->transTimer)) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - if (taosTmrReset(mndCalMqRebalance, MQ_TIMER_MS, pMnode, pMnode->timer, &pMnode->mqTimer)) { + if (taosTmrReset(mndCalMqRebalance, tsMaRebalanceMs, pMnode, pMnode->timer, &pMnode->mqTimer)) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp index aaf2c649e4..1fabf677c4 100644 --- a/source/dnode/mnode/impl/test/trans/trans2.cpp +++ b/source/dnode/mnode/impl/test/trans/trans2.cpp @@ -58,9 +58,12 @@ class MndTestTrans2 : public ::testing::Test { strcpy(opt.replicas[0].fqdn, "localhost"); opt.msgCb = msgCb; + tsTransPullupMs = 1000; + const char *mnodepath = "/tmp/mnode_test_trans"; taosRemoveDir(mnodepath); pMnode = mndOpen(mnodepath, &opt); + mndStart(pMnode); } static void SetUpTestSuite() { @@ -70,6 +73,7 @@ class MndTestTrans2 : public ::testing::Test { } static void TearDownTestSuite() { + mndStop(pMnode); mndClose(pMnode); walCleanUp(); taosCloseLog(); @@ -128,4 +132,6 @@ TEST_F(MndTestTrans2, 01_CbFunc) { EXPECT_EQ(CreateUser(acct_invalid, user2), 0); pUser2 = mndAcquireUser(pMnode, user2); ASSERT_EQ(pUser2, nullptr); + + mndTransPullup(pMnode); } From 0218def49a41c7391d762ce6ee544a02292b4747 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 4 May 2022 23:58:42 +0800 Subject: [PATCH 37/48] test: add action test for transaction --- source/dnode/mnode/impl/inc/mndTrans.h | 2 + source/dnode/mnode/impl/src/mndTrans.c | 45 ++++-- source/dnode/mnode/impl/test/trans/trans2.cpp | 128 ++++++++++++++++-- 3 files changed, 156 insertions(+), 19 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index 7644ec3c4c..2a011f414d 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -44,6 +44,8 @@ typedef void (*TransCbFp)(SMnode *pMnode, void *param, int32_t paramLen); int32_t mndInitTrans(SMnode *pMnode); void mndCleanupTrans(SMnode *pMnode); +STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId); +void mndReleaseTrans(SMnode *pMnode, STrans *pTrans); STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnType type, const SRpcMsg *pReq); void mndTransDrop(STrans *pTrans); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 6110d0be44..c5a31a6d85 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -537,7 +537,7 @@ static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOld, STrans *pNew) { return 0; } -static STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId) { +STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId) { STrans *pTrans = sdbAcquire(pMnode->pSdb, SDB_TRANS, &transId); if (pTrans == NULL) { terrno = TSDB_CODE_MND_TRANS_NOT_EXIST; @@ -545,7 +545,7 @@ static STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId) { return pTrans; } -static void mndReleaseTrans(SMnode *pMnode, STrans *pTrans) { +void mndReleaseTrans(SMnode *pMnode, STrans *pTrans) { SSdb *pSdb = pMnode->pSdb; sdbRelease(pSdb, pTrans); } @@ -919,27 +919,41 @@ static int32_t mndTransExecuteLogs(SMnode *pMnode, SArray *pArray) { if (arraySize == 0) return 0; + int32_t code = 0; for (int32_t i = 0; i < arraySize; ++i) { SSdbRaw *pRaw = taosArrayGetP(pArray, i); - int32_t code = sdbWriteWithoutFree(pSdb, pRaw); - if (code != 0) { - return code; + if (sdbWriteWithoutFree(pSdb, pRaw) != 0) { + code = ((terrno != 0) ? terrno : -1); } } - return 0; + terrno = code; + return code; } static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans) { - return mndTransExecuteLogs(pMnode, pTrans->redoLogs); + int32_t code = mndTransExecuteLogs(pMnode, pTrans->redoLogs); + if (code != 0) { + mError("failed to execute redoLogs since %s", terrstr()); + } + return code; } static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans) { - return mndTransExecuteLogs(pMnode, pTrans->undoLogs); + int32_t code = mndTransExecuteLogs(pMnode, pTrans->undoLogs); + if (code != 0) { + mError("failed to execute undoLogs since %s, return success", terrstr()); + } + + return 0; // return success in any case } static int32_t mndTransExecuteCommitLogs(SMnode *pMnode, STrans *pTrans) { - return mndTransExecuteLogs(pMnode, pTrans->commitLogs); + int32_t code = mndTransExecuteLogs(pMnode, pTrans->commitLogs); + if (code != 0) { + mError("failed to execute commitLogs since %s", terrstr()); + } + return code; } static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) { @@ -983,6 +997,7 @@ static int32_t mndTransSendActionMsg(SMnode *pMnode, STrans *pTrans, SArray *pAr pAction->msgReceived = 0; pAction->errCode = 0; } else { + if (terrno == TSDB_CODE_INVALID_PTR) rpcFreeCont(rpcMsg.pCont); mError("trans:%d, action:%d not send since %s", pTrans->id, action, terrstr()); return -1; } @@ -1029,11 +1044,19 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA } static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans) { - return mndTransExecuteActions(pMnode, pTrans, pTrans->redoActions); + int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->redoActions); + if (code != 0) { + mError("failed to execute redoActions since %s", terrstr()); + } + return code; } static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans) { - return mndTransExecuteActions(pMnode, pTrans, pTrans->undoActions); + int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->undoActions); + if (code != 0) { + mError("failed to execute undoActions since %s", terrstr()); + } + return code; } static bool mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans) { diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp index 1fabf677c4..e9d3b54c1d 100644 --- a/source/dnode/mnode/impl/test/trans/trans2.cpp +++ b/source/dnode/mnode/impl/test/trans/trans2.cpp @@ -86,7 +86,7 @@ class MndTestTrans2 : public ::testing::Test { void SetUp() override {} void TearDown() override {} - int32_t CreateUser(const char *acct, const char *user) { + int32_t CreateUserLog(const char *acct, const char *user) { SUserObj userObj = {0}; taosEncryptPass_c((uint8_t *)"taosdata", strlen("taosdata"), userObj.pass); tstrncpy(userObj.user, user, TSDB_USER_LEN); @@ -101,7 +101,11 @@ class MndTestTrans2 : public ::testing::Test { mndTransAppendRedolog(pTrans, pRedoRaw); sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); - char *param = strdup("====> test param <====="); + SSdbRaw *pUndoRaw = mndUserActionEncode(&userObj); + mndTransAppendUndolog(pTrans, pUndoRaw); + sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED); + + char *param = strdup("====> test log <====="); mndTransSetCb(pTrans, TEST_TRANS_START_FUNC, TEST_TRANS_STOP_FUNC, param, strlen(param) + 1); int32_t code = mndTransPrepare(pMnode, pTrans); @@ -109,29 +113,137 @@ class MndTestTrans2 : public ::testing::Test { return code; } + + int32_t CreateUserAction(const char *acct, const char *user, bool hasUndoAction) { + SUserObj userObj = {0}; + taosEncryptPass_c((uint8_t *)"taosdata", strlen("taosdata"), userObj.pass); + tstrncpy(userObj.user, user, TSDB_USER_LEN); + tstrncpy(userObj.acct, acct, TSDB_USER_LEN); + userObj.createdTime = taosGetTimestampMs(); + userObj.updateTime = userObj.createdTime; + userObj.superUser = 1; + + SRpcMsg rpcMsg = {0}; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, &rpcMsg); + SSdbRaw *pRedoRaw = mndUserActionEncode(&userObj); + mndTransAppendRedolog(pTrans, pRedoRaw); + sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); + + SSdbRaw *pUndoRaw = mndUserActionEncode(&userObj); + mndTransAppendUndolog(pTrans, pUndoRaw); + sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED); + + char *param = strdup("====> test action <====="); + mndTransSetCb(pTrans, TEST_TRANS_START_FUNC, TEST_TRANS_STOP_FUNC, param, strlen(param) + 1); + + { + STransAction action = {0}; + action.epSet.inUse = 0; + action.epSet.numOfEps = 1; + action.epSet.eps[0].port = 9040; + strcpy(action.epSet.eps[0].fqdn, "localhost"); + + int32_t contLen = 1024; + void *pReq = taosMemoryCalloc(1, contLen); + strcpy((char *)pReq, "hello world redo"); + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_DND_CREATE_MNODE; + action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; + mndTransAppendRedoAction(pTrans, &action); + } + + if (hasUndoAction) { + STransAction action = {0}; + action.epSet.inUse = 0; + action.epSet.numOfEps = 1; + action.epSet.eps[0].port = 9040; + strcpy(action.epSet.eps[0].fqdn, "localhost"); + + int32_t contLen = 1024; + void *pReq = taosMemoryCalloc(1, contLen); + strcpy((char *)pReq, "hello world undo"); + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_DND_CREATE_MNODE; + action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; + mndTransAppendUndoAction(pTrans, &action); + } + + int32_t code = mndTransPrepare(pMnode, pTrans); + mndTransDrop(pTrans); + + return code; + } }; SMnode *MndTestTrans2::pMnode; -TEST_F(MndTestTrans2, 01_CbFunc) { +TEST_F(MndTestTrans2, 01_Log) { const char *acct = "root"; const char *acct_invalid = "root1"; - const char *user1 = "test1"; - const char *user2 = "test2"; + const char *user1 = "log1"; + const char *user2 = "log2"; SUserObj *pUser1 = NULL; SUserObj *pUser2 = NULL; ASSERT_NE(pMnode, nullptr); - // create user success - EXPECT_EQ(CreateUser(acct, user1), 0); + EXPECT_EQ(CreateUserLog(acct, user1), 0); pUser1 = mndAcquireUser(pMnode, user1); ASSERT_NE(pUser1, nullptr); // failed to create user and rollback - EXPECT_EQ(CreateUser(acct_invalid, user2), 0); + EXPECT_EQ(CreateUserLog(acct_invalid, user2), 0); pUser2 = mndAcquireUser(pMnode, user2); ASSERT_EQ(pUser2, nullptr); mndTransPullup(pMnode); } + +TEST_F(MndTestTrans2, 02_Action) { + const char *acct = "root"; + const char *acct_invalid = "root1"; + const char *user1 = "action1"; + const char *user2 = "action2"; + SUserObj *pUser1 = NULL; + SUserObj *pUser2 = NULL; + STrans *pTrans = NULL; + int32_t transId = 4; + int32_t action = 0; + + ASSERT_NE(pMnode, nullptr); + + // failed to create user and rollback + EXPECT_EQ(CreateUserAction(acct, user1, false), 0); + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_EQ(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); + + // create user, and fake a response + EXPECT_EQ(CreateUserAction(acct, user1, true), 0); + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_NE(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); + + pTrans = mndAcquireTrans(pMnode, transId); + EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR); + EXPECT_EQ(pTrans->stage, TRN_STAGE_UNDO_ACTION); + EXPECT_EQ(pTrans->failedTimes, 1); + + STransAction *pAction = (STransAction *)taosArrayGet(pTrans->undoActions, action); + pAction->msgSent = 1; + + SNodeMsg rspMsg = {0}; + rspMsg.pNode = pMnode; + int64_t signature = transId; + signature = (signature << 32); + signature += action; + rspMsg.rpcMsg.ahandle = (void *)signature; + mndTransProcessRsp(&rspMsg); + mndReleaseTrans(pMnode, pTrans); + + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_EQ(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); +} From 43822106bc35ee92e930739244fdcd2d6c9bca2b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 5 May 2022 00:12:11 +0800 Subject: [PATCH 38/48] test: add unitest for transaction --- source/dnode/mnode/impl/test/trans/trans2.cpp | 109 ++++++++++++++---- 1 file changed, 84 insertions(+), 25 deletions(-) diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp index e9d3b54c1d..599f1882d4 100644 --- a/source/dnode/mnode/impl/test/trans/trans2.cpp +++ b/source/dnode/mnode/impl/test/trans/trans2.cpp @@ -114,7 +114,7 @@ class MndTestTrans2 : public ::testing::Test { return code; } - int32_t CreateUserAction(const char *acct, const char *user, bool hasUndoAction) { + int32_t CreateUserAction(const char *acct, const char *user, bool hasUndoAction, ETrnPolicy policy) { SUserObj userObj = {0}; taosEncryptPass_c((uint8_t *)"taosdata", strlen("taosdata"), userObj.pass); tstrncpy(userObj.user, user, TSDB_USER_LEN); @@ -124,7 +124,7 @@ class MndTestTrans2 : public ::testing::Test { userObj.superUser = 1; SRpcMsg rpcMsg = {0}; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, &rpcMsg); + STrans *pTrans = mndTransCreate(pMnode, policy, TRN_TYPE_CREATE_USER, &rpcMsg); SSdbRaw *pRedoRaw = mndUserActionEncode(&userObj); mndTransAppendRedolog(pTrans, pRedoRaw); sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); @@ -209,41 +209,100 @@ TEST_F(MndTestTrans2, 02_Action) { SUserObj *pUser1 = NULL; SUserObj *pUser2 = NULL; STrans *pTrans = NULL; - int32_t transId = 4; + int32_t transId = 0; int32_t action = 0; ASSERT_NE(pMnode, nullptr); // failed to create user and rollback - EXPECT_EQ(CreateUserAction(acct, user1, false), 0); + EXPECT_EQ(CreateUserAction(acct, user1, false, TRN_POLICY_ROLLBACK), 0); pUser1 = mndAcquireUser(pMnode, user1); ASSERT_EQ(pUser1, nullptr); mndReleaseUser(pMnode, pUser1); // create user, and fake a response - EXPECT_EQ(CreateUserAction(acct, user1, true), 0); - pUser1 = mndAcquireUser(pMnode, user1); - ASSERT_NE(pUser1, nullptr); - mndReleaseUser(pMnode, pUser1); + { + EXPECT_EQ(CreateUserAction(acct, user1, true, TRN_POLICY_ROLLBACK), 0); + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_NE(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); - pTrans = mndAcquireTrans(pMnode, transId); - EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR); - EXPECT_EQ(pTrans->stage, TRN_STAGE_UNDO_ACTION); - EXPECT_EQ(pTrans->failedTimes, 1); + transId = 4; + pTrans = mndAcquireTrans(pMnode, transId); + EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR); + EXPECT_EQ(pTrans->stage, TRN_STAGE_UNDO_ACTION); + EXPECT_EQ(pTrans->failedTimes, 1); - STransAction *pAction = (STransAction *)taosArrayGet(pTrans->undoActions, action); - pAction->msgSent = 1; + STransAction *pAction = (STransAction *)taosArrayGet(pTrans->undoActions, action); + pAction->msgSent = 1; - SNodeMsg rspMsg = {0}; - rspMsg.pNode = pMnode; - int64_t signature = transId; - signature = (signature << 32); - signature += action; - rspMsg.rpcMsg.ahandle = (void *)signature; - mndTransProcessRsp(&rspMsg); - mndReleaseTrans(pMnode, pTrans); + SNodeMsg rspMsg = {0}; + rspMsg.pNode = pMnode; + int64_t signature = transId; + signature = (signature << 32); + signature += action; + rspMsg.rpcMsg.ahandle = (void *)signature; + mndTransProcessRsp(&rspMsg); + mndReleaseTrans(pMnode, pTrans); - pUser1 = mndAcquireUser(pMnode, user1); - ASSERT_EQ(pUser1, nullptr); - mndReleaseUser(pMnode, pUser1); + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_EQ(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); + } + + { + EXPECT_EQ(CreateUserAction(acct, user1, false, TRN_POLICY_RETRY), 0); + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_NE(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); + + { + transId = 5; + pTrans = mndAcquireTrans(pMnode, transId); + EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR); + EXPECT_EQ(pTrans->stage, TRN_STAGE_REDO_ACTION); + EXPECT_EQ(pTrans->failedTimes, 1); + + STransAction *pAction = (STransAction *)taosArrayGet(pTrans->redoActions, action); + pAction->msgSent = 1; + + SNodeMsg rspMsg = {0}; + rspMsg.pNode = pMnode; + int64_t signature = transId; + signature = (signature << 32); + signature += action; + rspMsg.rpcMsg.ahandle = (void *)signature; + rspMsg.rpcMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; + mndTransProcessRsp(&rspMsg); + mndReleaseTrans(pMnode, pTrans); + + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_NE(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); + } + + { + transId = 5; + pTrans = mndAcquireTrans(pMnode, transId); + EXPECT_EQ(pTrans->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); + EXPECT_EQ(pTrans->stage, TRN_STAGE_REDO_ACTION); + EXPECT_EQ(pTrans->failedTimes, 2); + + STransAction *pAction = (STransAction *)taosArrayGet(pTrans->redoActions, action); + pAction->msgSent = 1; + + SNodeMsg rspMsg = {0}; + rspMsg.pNode = pMnode; + int64_t signature = transId; + signature = (signature << 32); + signature += action; + rspMsg.rpcMsg.ahandle = (void *)signature; + mndTransProcessRsp(&rspMsg); + mndReleaseTrans(pMnode, pTrans); + + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_NE(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); + } + } } From 00b29f06770d97476e25ab12f3ad08bacf14ad51 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 5 May 2022 09:36:11 +0800 Subject: [PATCH 39/48] feature/qnode --- tests/script/api/batchprepare.c | 828 ++++---------------------------- 1 file changed, 94 insertions(+), 734 deletions(-) diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 7bea19ae90..828216d1d3 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -462,6 +462,29 @@ void generateQuerySQL(BindData *data, int32_t tblIdx) { } } +void generateErrorSQL(BindData *data, int32_t tblIdx) { + int32_t len = 0; + data->sql = taosMemoryCalloc(1, 1024); + + switch(tblIdx) { + case 0: + len = sprintf(data->sql, "insert into %s%d values (?, 1)", bpTbPrefix, tblIdx); + break; + case 1: + len = sprintf(data->sql, "select * from ?"); + break; + case 2: + len = sprintf(data->sql, "select * from %s%d where ? = ?", bpTbPrefix, tblIdx); + break; + default: + len = sprintf(data->sql, "select count(*) from %s%d group by ?", bpTbPrefix, tblIdx); + break; + } + + if (gCaseCtrl.printStmtSql) { + printf("\tSTMT SQL: %s\n", data->sql); + } +} void generateDataType(BindData *data, int32_t bindIdx, int32_t colIdx, int32_t *dataType) { if (bindIdx < gCurCase->bindColNum) { @@ -712,6 +735,8 @@ int32_t prepareQueryData(BindData *data, int32_t tblIdx) { } + + void destroyData(BindData *data) { taosMemoryFree(data->tsData); taosMemoryFree(data->boolData); @@ -1354,6 +1379,7 @@ int insertMPMETest1(TAOS_STMT *stmt, TAOS *taos) { return 0; } +/* select * from table */ int querySUBTTest1(TAOS_STMT *stmt, TAOS *taos) { BindData data = {0}; @@ -1399,6 +1425,74 @@ int querySUBTTest1(TAOS_STMT *stmt, TAOS *taos) { return 0; } +/* value in query sql */ +int querySUBTTest2(TAOS_STMT *stmt, TAOS *taos) { + BindData data = {0}; + + for (int32_t t = 0; t< gCurCase->tblNum; ++t) { + memset(&data, 0, sizeof(data)); + prepareQueryData(&data, t); + + int code = taos_stmt_prepare(stmt, data.sql, 0); + if (code != 0){ + printf("!!!failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + for (int32_t n = 0; n< (gCurCase->rowNum/gCurCase->bindRowNum); ++n) { + bpCheckIsInsert(stmt, 0); + + if (gCaseCtrl.checkParamNum) { + bpCheckParamNum(stmt); + } + + if (bpBindParam(stmt, data.pBind + n * gCurCase->bindColNum)) { + exit(1); + } + + if (taos_stmt_add_batch(stmt)) { + printf("!!!taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + if (taos_stmt_execute(stmt) != 0) { + printf("!!!taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + bpCheckQueryResult(stmt, taos, data.sql, data.pBind + n * gCurCase->bindColNum); + } + + bpCheckIsInsert(stmt, 0); + + destroyData(&data); + } + + return 0; +} + + +int errorSQLTest1(TAOS_STMT *stmt, TAOS *taos) { + BindData data = {0}; + + for (int32_t t = 0; t< gCurCase->tblNum; ++t) { + memset(&data, 0, sizeof(data)); + generateErrorSQL(&data, t); + + int code = taos_stmt_prepare(stmt, data.sql, 0); + if (code != 0){ + printf("*taos_stmt_prepare error as expected, error:%s\n", taos_stmt_errstr(stmt)); + } else { + printf("!!!taos_stmt_prepare succeed, which should be error\n"); + exit(1); + } + + destroyData(&data); + } + + return 0; +} + #if 0 @@ -1423,153 +1517,6 @@ int stmt_funcb_autoctb1(TAOS_STMT *stmt) { TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); -// int one_null = 1; - int one_not_null = 0; - - char* is_null = taosMemoryMalloc(sizeof(char) * 10); - char* no_null = taosMemoryMalloc(sizeof(char) * 10); - - for (int i = 0; i < 10; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 10; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[10*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 10; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 10; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 10; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 10; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 10; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 10; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 10; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 10; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 10; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 10; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 10; ++i) { - v.ts[i] = tts + i; - } - - - for (int i = 0; i < 1; ++i) { - tags[i+0].buffer_type = TSDB_DATA_TYPE_INT; - tags[i+0].buffer = v.v4; - tags[i+0].is_null = &one_not_null; - tags[i+0].length = NULL; - - tags[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - tags[i+1].buffer = v.b; - tags[i+1].is_null = &one_not_null; - tags[i+1].length = NULL; - - tags[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - tags[i+2].buffer = v.v1; - tags[i+2].is_null = &one_not_null; - tags[i+2].length = NULL; - - tags[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - tags[i+3].buffer = v.v2; - tags[i+3].is_null = &one_not_null; - tags[i+3].length = NULL; - - tags[i+4].buffer_type = TSDB_DATA_TYPE_BIGINT; - tags[i+4].buffer = v.v8; - tags[i+4].is_null = &one_not_null; - tags[i+4].length = NULL; - - tags[i+5].buffer_type = TSDB_DATA_TYPE_FLOAT; - tags[i+5].buffer = v.f4; - tags[i+5].is_null = &one_not_null; - tags[i+5].length = NULL; - - tags[i+6].buffer_type = TSDB_DATA_TYPE_DOUBLE; - tags[i+6].buffer = v.f8; - tags[i+6].is_null = &one_not_null; - tags[i+6].length = NULL; - - tags[i+7].buffer_type = TSDB_DATA_TYPE_BINARY; - tags[i+7].buffer = v.bin; - tags[i+7].is_null = &one_not_null; - tags[i+7].length = (uintptr_t *)lb; - - tags[i+8].buffer_type = TSDB_DATA_TYPE_NCHAR; - tags[i+8].buffer = v.bin; - tags[i+8].is_null = &one_not_null; - tags[i+8].length = (uintptr_t *)lb; - } - - unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; @@ -2383,153 +2330,6 @@ int stmt_funcb_autoctb_e2(TAOS_STMT *stmt) { TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); -// int one_null = 1; - int one_not_null = 0; - - char* is_null = taosMemoryMalloc(sizeof(char) * 10); - char* no_null = taosMemoryMalloc(sizeof(char) * 10); - - for (int i = 0; i < 10; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 10; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[10*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 10; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 10; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 10; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 10; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 10; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 10; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 10; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 10; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 10; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 10; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 10; ++i) { - v.ts[i] = tts + i; - } - - - for (int i = 0; i < 1; ++i) { - tags[i+0].buffer_type = TSDB_DATA_TYPE_INT; - tags[i+0].buffer = v.v4; - tags[i+0].is_null = &one_not_null; - tags[i+0].length = NULL; - - tags[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - tags[i+1].buffer = v.b; - tags[i+1].is_null = &one_not_null; - tags[i+1].length = NULL; - - tags[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - tags[i+2].buffer = v.v1; - tags[i+2].is_null = &one_not_null; - tags[i+2].length = NULL; - - tags[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - tags[i+3].buffer = v.v2; - tags[i+3].is_null = &one_not_null; - tags[i+3].length = NULL; - - tags[i+4].buffer_type = TSDB_DATA_TYPE_BIGINT; - tags[i+4].buffer = v.v8; - tags[i+4].is_null = &one_not_null; - tags[i+4].length = NULL; - - tags[i+5].buffer_type = TSDB_DATA_TYPE_FLOAT; - tags[i+5].buffer = v.f4; - tags[i+5].is_null = &one_not_null; - tags[i+5].length = NULL; - - tags[i+6].buffer_type = TSDB_DATA_TYPE_DOUBLE; - tags[i+6].buffer = v.f8; - tags[i+6].is_null = &one_not_null; - tags[i+6].length = NULL; - - tags[i+7].buffer_type = TSDB_DATA_TYPE_BINARY; - tags[i+7].buffer = v.bin; - tags[i+7].is_null = &one_not_null; - tags[i+7].length = (uintptr_t *)lb; - - tags[i+8].buffer_type = TSDB_DATA_TYPE_NCHAR; - tags[i+8].buffer = v.bin; - tags[i+8].is_null = &one_not_null; - tags[i+8].length = (uintptr_t *)lb; - } - - unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; @@ -2600,152 +2400,6 @@ int stmt_funcb_autoctb_e3(TAOS_STMT *stmt) { TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); -// int one_null = 1; - int one_not_null = 0; - - char* is_null = taosMemoryMalloc(sizeof(char) * 10); - char* no_null = taosMemoryMalloc(sizeof(char) * 10); - - for (int i = 0; i < 10; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 10; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[10*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 10; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 10; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 10; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 10; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 10; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 10; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 10; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 10; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 10; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 10; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 10; ++i) { - v.ts[i] = tts + i; - } - - - for (int i = 0; i < 1; ++i) { - tags[i+0].buffer_type = TSDB_DATA_TYPE_INT; - tags[i+0].buffer = v.v4; - tags[i+0].is_null = &one_not_null; - tags[i+0].length = NULL; - - tags[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - tags[i+1].buffer = v.b; - tags[i+1].is_null = &one_not_null; - tags[i+1].length = NULL; - - tags[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - tags[i+2].buffer = v.v1; - tags[i+2].is_null = &one_not_null; - tags[i+2].length = NULL; - - tags[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - tags[i+3].buffer = v.v2; - tags[i+3].is_null = &one_not_null; - tags[i+3].length = NULL; - - tags[i+4].buffer_type = TSDB_DATA_TYPE_BIGINT; - tags[i+4].buffer = v.v8; - tags[i+4].is_null = &one_not_null; - tags[i+4].length = NULL; - - tags[i+5].buffer_type = TSDB_DATA_TYPE_FLOAT; - tags[i+5].buffer = v.f4; - tags[i+5].is_null = &one_not_null; - tags[i+5].length = NULL; - - tags[i+6].buffer_type = TSDB_DATA_TYPE_DOUBLE; - tags[i+6].buffer = v.f8; - tags[i+6].is_null = &one_not_null; - tags[i+6].length = NULL; - - tags[i+7].buffer_type = TSDB_DATA_TYPE_BINARY; - tags[i+7].buffer = v.bin; - tags[i+7].is_null = &one_not_null; - tags[i+7].length = (uintptr_t *)lb; - - tags[i+8].buffer_type = TSDB_DATA_TYPE_NCHAR; - tags[i+8].buffer = v.bin; - tags[i+8].is_null = &one_not_null; - tags[i+8].length = (uintptr_t *)lb; - } - unsigned long long starttime = taosGetTimestampUs(); @@ -2815,153 +2469,6 @@ int stmt_funcb_autoctb_e4(TAOS_STMT *stmt) { TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); -// int one_null = 1; - int one_not_null = 0; - - char* is_null = taosMemoryMalloc(sizeof(char) * 10); - char* no_null = taosMemoryMalloc(sizeof(char) * 10); - - for (int i = 0; i < 10; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 10; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[10*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 10; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 10; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 10; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 10; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 10; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 10; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 10; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 10; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 10; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 10; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 10; ++i) { - v.ts[i] = tts + i; - } - - - for (int i = 0; i < 1; ++i) { - tags[i+0].buffer_type = TSDB_DATA_TYPE_INT; - tags[i+0].buffer = v.v4; - tags[i+0].is_null = &one_not_null; - tags[i+0].length = NULL; - - tags[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - tags[i+1].buffer = v.b; - tags[i+1].is_null = &one_not_null; - tags[i+1].length = NULL; - - tags[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - tags[i+2].buffer = v.v1; - tags[i+2].is_null = &one_not_null; - tags[i+2].length = NULL; - - tags[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - tags[i+3].buffer = v.v2; - tags[i+3].is_null = &one_not_null; - tags[i+3].length = NULL; - - tags[i+4].buffer_type = TSDB_DATA_TYPE_BIGINT; - tags[i+4].buffer = v.v8; - tags[i+4].is_null = &one_not_null; - tags[i+4].length = NULL; - - tags[i+5].buffer_type = TSDB_DATA_TYPE_FLOAT; - tags[i+5].buffer = v.f4; - tags[i+5].is_null = &one_not_null; - tags[i+5].length = NULL; - - tags[i+6].buffer_type = TSDB_DATA_TYPE_DOUBLE; - tags[i+6].buffer = v.f8; - tags[i+6].is_null = &one_not_null; - tags[i+6].length = NULL; - - tags[i+7].buffer_type = TSDB_DATA_TYPE_BINARY; - tags[i+7].buffer = v.bin; - tags[i+7].is_null = &one_not_null; - tags[i+7].length = (uintptr_t *)lb; - - tags[i+8].buffer_type = TSDB_DATA_TYPE_NCHAR; - tags[i+8].buffer = v.bin; - tags[i+8].is_null = &one_not_null; - tags[i+8].length = (uintptr_t *)lb; - } - - unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; @@ -3042,153 +2549,6 @@ int stmt_funcb_autoctb_e5(TAOS_STMT *stmt) { TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); -// int one_null = 1; - int one_not_null = 0; - - char* is_null = taosMemoryMalloc(sizeof(char) * 10); - char* no_null = taosMemoryMalloc(sizeof(char) * 10); - - for (int i = 0; i < 10; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 10; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[10*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 10; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 10; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 10; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 10; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 10; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 10; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 10; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 10; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 10; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 10; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 10; ++i) { - v.ts[i] = tts + i; - } - - - for (int i = 0; i < 1; ++i) { - tags[i+0].buffer_type = TSDB_DATA_TYPE_INT; - tags[i+0].buffer = v.v4; - tags[i+0].is_null = &one_not_null; - tags[i+0].length = NULL; - - tags[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - tags[i+1].buffer = v.b; - tags[i+1].is_null = &one_not_null; - tags[i+1].length = NULL; - - tags[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - tags[i+2].buffer = v.v1; - tags[i+2].is_null = &one_not_null; - tags[i+2].length = NULL; - - tags[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - tags[i+3].buffer = v.v2; - tags[i+3].is_null = &one_not_null; - tags[i+3].length = NULL; - - tags[i+4].buffer_type = TSDB_DATA_TYPE_BIGINT; - tags[i+4].buffer = v.v8; - tags[i+4].is_null = &one_not_null; - tags[i+4].length = NULL; - - tags[i+5].buffer_type = TSDB_DATA_TYPE_FLOAT; - tags[i+5].buffer = v.f4; - tags[i+5].is_null = &one_not_null; - tags[i+5].length = NULL; - - tags[i+6].buffer_type = TSDB_DATA_TYPE_DOUBLE; - tags[i+6].buffer = v.f8; - tags[i+6].is_null = &one_not_null; - tags[i+6].length = NULL; - - tags[i+7].buffer_type = TSDB_DATA_TYPE_BINARY; - tags[i+7].buffer = v.bin; - tags[i+7].is_null = &one_not_null; - tags[i+7].length = (uintptr_t *)lb; - - tags[i+8].buffer_type = TSDB_DATA_TYPE_NCHAR; - tags[i+8].buffer = v.bin; - tags[i+8].is_null = &one_not_null; - tags[i+8].length = (uintptr_t *)lb; - } - - unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; From 7c5c1c32ee48b3a49728f654522e06f2936d7687 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 5 May 2022 03:17:34 +0000 Subject: [PATCH 40/48] implement TDB upsert --- source/libs/tdb/inc/tdb.h | 2 +- source/libs/tdb/src/db/tdbBtree.c | 148 ++++++++++++++++++++---------- source/libs/tdb/src/db/tdbDb.c | 5 +- source/libs/tdb/src/inc/tdbInt.h | 3 +- source/libs/tdb/test/tdbTest.cpp | 69 +++++++++++++- 5 files changed, 170 insertions(+), 57 deletions(-) diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index cab242d5b3..9e79ec1d89 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -42,7 +42,7 @@ int tdbDbClose(TDB *pDb); int tdbDbDrop(TDB *pDb); int tdbDbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn); int tdbDbDelete(TDB *pDb, const void *pKey, int kLen, TXN *pTxn); -int tdbUpsert(TDB *pTDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn); +int tdbDbUpsert(TDB *pDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn); int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen); int tdbDbPGet(TDB *pDb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index ed7aed552a..0d13ec4d2b 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -138,67 +138,25 @@ int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, in } if (btc.idx == -1) { - idx = 0; + btc.idx = 0; } else { if (c > 0) { - idx = btc.idx + 1; - } else if (c < 0) { - idx = btc.idx; - } else { - // TDB does NOT allow same key - tdbBtcClose(&btc); + btc.idx++; + } else if (c == 0) { + // dup key not allowed ASSERT(0); return -1; } } - // make sure enough space to hold the cell - szBuf = kLen + vLen + 14; - pBuf = tdbRealloc(pBt->pBuf, pBt->pageSize > szBuf ? szBuf : pBt->pageSize); - if (pBuf == NULL) { - tdbBtcClose(&btc); - ASSERT(0); - return -1; - } - pBt->pBuf = pBuf; - pCell = (SCell *)pBt->pBuf; - - // encode cell - ret = tdbBtreeEncodeCell(btc.pPage, pKey, kLen, pVal, vLen, pCell, &szCell); + ret = tdbBtcUpsert(&btc, pKey, kLen, pVal, vLen, 1); if (ret < 0) { - tdbBtcClose(&btc); ASSERT(0); - return -1; - } - - // mark the page dirty - ret = tdbPagerWrite(pBt->pPager, btc.pPage); - if (ret < 0) { tdbBtcClose(&btc); - ASSERT(0); return -1; } - // insert the cell - ret = tdbPageInsertCell(btc.pPage, idx, pCell, szCell, 0); - if (ret < 0) { - tdbBtcClose(&btc); - ASSERT(0); - return -1; - } - - // check if need balance - if (btc.pPage->nOverflow > 0) { - ret = tdbBtreeBalance(&btc); - if (ret < 0) { - tdbBtcClose(&btc); - ASSERT(0); - return -1; - } - } - tdbBtcClose(&btc); - return 0; } @@ -232,6 +190,41 @@ int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn) { return 0; } +int tdbBtreeUpsert(SBTree *pBt, const void *pKey, int nKey, const void *pData, int nData, TXN *pTxn) { + SBTC btc; + int c; + int ret; + + tdbBtcOpen(&btc, pBt, pTxn); + + // move the cursor + ret = tdbBtcMoveTo(&btc, pKey, nKey, &c); + if (ret < 0) { + ASSERT(0); + tdbBtcClose(&btc); + return -1; + } + + if (btc.idx == -1) { + btc.idx = 0; + c = 1; + } else { + if (c > 0) { + btc.idx = btc.idx + 1; + } + } + + ret = tdbBtcUpsert(&btc, pKey, nKey, pData, nData, c); + if (ret < 0) { + ASSERT(0); + tdbBtcClose(&btc); + return -1; + } + + tdbBtcClose(&btc); + return 0; +} + int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen) { return tdbBtreePGet(pBt, pKey, kLen, NULL, NULL, ppVal, vLen); } @@ -1475,9 +1468,64 @@ int tdbBtcDelete(SBTC *pBtc) { return 0; } -int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pVal, int vLen) { - ASSERT(0); - // TODO +int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int nData, int insert) { + SCell *pCell; + int szCell; + int nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage); + int szBuf; + void *pBuf; + int ret; + + ASSERT(pBtc->idx >= 0); + + // alloc space + szBuf = kLen + nData + 14; + pBuf = tdbRealloc(pBtc->pBt->pBuf, pBtc->pBt->pageSize > szBuf ? szBuf : pBtc->pBt->pageSize); + if (pBuf == NULL) { + ASSERT(0); + return -1; + } + pBtc->pBt->pBuf = pBuf; + pCell = (SCell *)pBtc->pBt->pBuf; + + // encode cell + ret = tdbBtreeEncodeCell(pBtc->pPage, pKey, kLen, pData, nData, pCell, &szCell); + if (ret < 0) { + ASSERT(0); + return -1; + } + + // mark dirty + ret = tdbPagerWrite(pBtc->pBt->pPager, pBtc->pPage); + if (ret < 0) { + ASSERT(0); + return -1; + } + + // insert or update + if (insert) { + ASSERT(pBtc->idx <= nCells); + + ret = tdbPageInsertCell(pBtc->pPage, pBtc->idx, pCell, szCell, 0); + } else { + ASSERT(pBtc->idx < nCells); + + ret = tdbPageUpdateCell(pBtc->pPage, pBtc->idx, pCell, szCell); + } + if (ret < 0) { + ASSERT(0); + return -1; + } + + // check balance + if (pBtc->pPage->nOverflow > 0) { + ret = tdbBtreeBalance(pBtc); + if (ret < 0) { + ASSERT(0); + return -1; + } + } + return 0; } diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 3358aefec0..c31441d849 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -81,9 +81,8 @@ int tdbDbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int va int tdbDbDelete(TDB *pDb, const void *pKey, int kLen, TXN *pTxn) { return tdbBtreeDelete(pDb->pBt, pKey, kLen, pTxn); } -int tdbUpsert(TDB *pTDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn) { - // TODO - return 0; +int tdbDbUpsert(TDB *pDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn) { + return tdbBtreeUpsert(pDb->pBt, pKey, kLen, pVal, vLen, pTxn); } int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) { diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index e18952505c..76dacf7b84 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -129,6 +129,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, tdb_cmpr_fn_t kcmpr, SBT int tdbBtreeClose(SBTree *pBt); int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn); int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn); +int tdbBtreeUpsert(SBTree *pBt, const void *pKey, int nKey, const void *pData, int nData, TXN *pTxn); int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen); int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); @@ -143,7 +144,7 @@ int tdbBtcMoveToPrev(SBTC *pBtc); int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen); int tdbBtcGet(SBTC *pBtc, const void **ppKey, int *kLen, const void **ppVal, int *vLen); int tdbBtcDelete(SBTC *pBtc); -int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pVal, int vLen); +int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int nData, int insert); // tdbPager.c ==================================== diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 994e53cd9d..e575ac156f 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -115,7 +115,7 @@ static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, in return cret; } -TEST(tdb_test, simple_test) { +TEST(tdb_test, simple_insert1) { int ret; TENV *pEnv; TDB *pDb; @@ -235,7 +235,7 @@ TEST(tdb_test, simple_test) { GTEST_ASSERT_EQ(ret, 0); } -TEST(tdb_test, simple_test2) { +TEST(tdb_test, simple_insert2) { int ret; TENV *pEnv; TDB *pDb; @@ -413,6 +413,71 @@ TEST(tdb_test, simple_delete1) { closePool(pPool); + tdbDbClose(pDb); + tdbEnvClose(pEnv); +} + +TEST(tdb_test, simple_upsert1) { + int ret; + TENV *pEnv; + TDB *pDb; + int nData = 100000; + char key[64]; + char data[64]; + void *pData = NULL; + SPoolMem *pPool; + TXN txn; + + taosRemoveDir("tdb"); + + // open env + ret = tdbEnvOpen("tdb", 4096, 64, &pEnv); + GTEST_ASSERT_EQ(ret, 0); + + // open database + ret = tdbDbOpen("db.db", -1, -1, NULL, pEnv, &pDb); + GTEST_ASSERT_EQ(ret, 0); + + pPool = openPool(); + // insert some data + tdbTxnOpen(&txn, 0, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + tdbBegin(pEnv, &txn); + + for (int iData = 0; iData < nData; iData++) { + sprintf(key, "key%d", iData); + sprintf(data, "data%d", iData); + ret = tdbDbInsert(pDb, key, strlen(key), data, strlen(data), &txn); + GTEST_ASSERT_EQ(ret, 0); + } + + // query the data + for (int iData = 0; iData < nData; iData++) { + sprintf(key, "key%d", iData); + sprintf(data, "data%d", iData); + ret = tdbDbGet(pDb, key, strlen(key), &pData, &nData); + GTEST_ASSERT_EQ(ret, 0); + GTEST_ASSERT_EQ(memcmp(pData, data, nData), 0); + } + + // upsert some data + for (int iData = 0; iData < nData; iData++) { + sprintf(key, "key%d", iData); + sprintf(data, "data%d-u", iData); + ret = tdbDbUpsert(pDb, key, strlen(key), data, strlen(data), &txn); + GTEST_ASSERT_EQ(ret, 0); + } + + tdbCommit(pEnv, &txn); + + // query the data + for (int iData = 0; iData < nData; iData++) { + sprintf(key, "key%d", iData); + sprintf(data, "data%d-u", iData); + ret = tdbDbGet(pDb, key, strlen(key), &pData, &nData); + GTEST_ASSERT_EQ(ret, 0); + GTEST_ASSERT_EQ(memcmp(pData, data, nData), 0); + } + tdbDbClose(pDb); tdbEnvClose(pEnv); } \ No newline at end of file From c87eb6cc479353f85a60558d418fa0e736176f76 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 5 May 2022 03:24:00 +0000 Subject: [PATCH 41/48] refact --- source/libs/tdb/inc/tdb.h | 4 +--- source/libs/tdb/src/db/tdbDb.c | 16 ++++------------ 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 9e79ec1d89..18f8ddec5c 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -55,11 +55,9 @@ int tdbDbcMoveToLast(TDBC *pDbc); int tdbDbcMoveToNext(TDBC *pDbc); int tdbDbcMoveToPrev(TDBC *pDbc); int tdbDbcGet(TDBC *pDbc, const void **ppKey, int *pkLen, const void **ppVal, int *pvLen); - -int tdbDbcPut(TDBC *pDbc, const void *pKey, int keyLen, const void *pVal, int valLen); -int tdbDbcUpdate(TDBC *pDbc, const void *pKey, int kLen, const void *pVal, int vLen); int tdbDbcDelete(TDBC *pDbc); int tdbDbcNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen); +int tdbDbcUpsert(TDBC *pDbc, const void *pKey, int nKey, const void *pData, int nData, int insert); // TXN #define TDB_TXN_WRITE 0x1 diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index c31441d849..553bb2c646 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -123,24 +123,16 @@ int tdbDbcGet(TDBC *pDbc, const void **ppKey, int *pkLen, const void **ppVal, in return tdbBtcGet(&pDbc->btc, ppKey, pkLen, ppVal, pvLen); } -int tdbDbcPut(TDBC *pDbc, const void *pKey, int keyLen, const void *pVal, int valLen) { - // TODO - ASSERT(0); - return 0; -} - -int tdbDbcUpdate(TDBC *pDbc, const void *pKey, int kLen, const void *pVal, int vLen) { - // TODO - ASSERT(0); - return 0; -} - int tdbDbcDelete(TDBC *pDbc) { return tdbBtcDelete(&pDbc->btc); } int tdbDbcNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen) { return tdbBtreeNext(&pDbc->btc, ppKey, kLen, ppVal, vLen); } +int tdbDbcUpsert(TDBC *pDbc, const void *pKey, int nKey, const void *pData, int nData, int insert) { + return tdbBtcUpsert(&pDbc->btc, pKey, nKey, pData, nData, insert); +} + int tdbDbcClose(TDBC *pDbc) { if (pDbc) { tdbBtcClose(&pDbc->btc); From af7000e3dc2ae6cae7583f0f4ee07f47b8053323 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 5 May 2022 11:48:52 +0800 Subject: [PATCH 42/48] feat: update taos-tools for 3.0 (#12112) [TD-13052] --- tools/taos-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/taos-tools b/tools/taos-tools index bf6c766986..2f3dfddd4d 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit bf6c766986c61ff4fc80421fdea682a8fd4b5b32 +Subproject commit 2f3dfddd4d9a869e706ba3cf98fb6d769404cd7c From 5191a1ca91e74b0f357f73a0190e559c8aadcf38 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 5 May 2022 13:13:23 +0800 Subject: [PATCH 43/48] test: add rollback test for transaction --- source/dnode/mnode/impl/src/mndTrans.c | 3 + source/dnode/mnode/impl/test/trans/trans2.cpp | 121 +++++++++++++----- 2 files changed, 94 insertions(+), 30 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index c5a31a6d85..9144de82b3 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -997,6 +997,9 @@ static int32_t mndTransSendActionMsg(SMnode *pMnode, STrans *pTrans, SArray *pAr pAction->msgReceived = 0; pAction->errCode = 0; } else { + pAction->msgSent = 0; + pAction->msgReceived = 0; + pAction->errCode = terrno; if (terrno == TSDB_CODE_INVALID_PTR) rpcFreeCont(rpcMsg.pCont); mError("trans:%d, action:%d not send since %s", pTrans->id, action, terrstr()); return -1; diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp index 599f1882d4..e81827041b 100644 --- a/source/dnode/mnode/impl/test/trans/trans2.cpp +++ b/source/dnode/mnode/impl/test/trans/trans2.cpp @@ -170,6 +170,12 @@ class MndTestTrans2 : public ::testing::Test { mndTransAppendUndoAction(pTrans, &action); } + { + void *pRsp = taosMemoryCalloc(1, 256); + strcpy((char *)pRsp, "simple rsponse"); + mndTransSetRpcRsp(pTrans, pRsp, 256); + } + int32_t code = mndTransPrepare(pMnode, pTrans); mndTransDrop(pTrans); @@ -214,40 +220,42 @@ TEST_F(MndTestTrans2, 02_Action) { ASSERT_NE(pMnode, nullptr); - // failed to create user and rollback - EXPECT_EQ(CreateUserAction(acct, user1, false, TRN_POLICY_ROLLBACK), 0); - pUser1 = mndAcquireUser(pMnode, user1); - ASSERT_EQ(pUser1, nullptr); - mndReleaseUser(pMnode, pUser1); - - // create user, and fake a response { - EXPECT_EQ(CreateUserAction(acct, user1, true, TRN_POLICY_ROLLBACK), 0); - pUser1 = mndAcquireUser(pMnode, user1); - ASSERT_NE(pUser1, nullptr); - mndReleaseUser(pMnode, pUser1); - - transId = 4; - pTrans = mndAcquireTrans(pMnode, transId); - EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR); - EXPECT_EQ(pTrans->stage, TRN_STAGE_UNDO_ACTION); - EXPECT_EQ(pTrans->failedTimes, 1); - - STransAction *pAction = (STransAction *)taosArrayGet(pTrans->undoActions, action); - pAction->msgSent = 1; - - SNodeMsg rspMsg = {0}; - rspMsg.pNode = pMnode; - int64_t signature = transId; - signature = (signature << 32); - signature += action; - rspMsg.rpcMsg.ahandle = (void *)signature; - mndTransProcessRsp(&rspMsg); - mndReleaseTrans(pMnode, pTrans); - + // failed to create user and rollback + EXPECT_EQ(CreateUserAction(acct, user1, false, TRN_POLICY_ROLLBACK), 0); pUser1 = mndAcquireUser(pMnode, user1); ASSERT_EQ(pUser1, nullptr); mndReleaseUser(pMnode, pUser1); + + // create user, and fake a response + { + EXPECT_EQ(CreateUserAction(acct, user1, true, TRN_POLICY_ROLLBACK), 0); + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_NE(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); + + transId = 4; + pTrans = mndAcquireTrans(pMnode, transId); + EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR); + EXPECT_EQ(pTrans->stage, TRN_STAGE_UNDO_ACTION); + EXPECT_EQ(pTrans->failedTimes, 1); + + STransAction *pAction = (STransAction *)taosArrayGet(pTrans->undoActions, action); + pAction->msgSent = 1; + + SNodeMsg rspMsg = {0}; + rspMsg.pNode = pMnode; + int64_t signature = transId; + signature = (signature << 32); + signature += action; + rspMsg.rpcMsg.ahandle = (void *)signature; + mndTransProcessRsp(&rspMsg); + mndReleaseTrans(pMnode, pTrans); + + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_EQ(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); + } } { @@ -305,4 +313,57 @@ TEST_F(MndTestTrans2, 02_Action) { mndReleaseUser(pMnode, pUser1); } } + + { + EXPECT_EQ(CreateUserAction(acct, user2, true, TRN_POLICY_ROLLBACK), 0); + SUserObj *pUser2 = (SUserObj *)sdbAcquire(pMnode->pSdb, SDB_USER, user2); + ASSERT_NE(pUser2, nullptr); + mndReleaseUser(pMnode, pUser2); + + { + transId = 6; + pTrans = mndAcquireTrans(pMnode, transId); + EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR); + EXPECT_EQ(pTrans->stage, TRN_STAGE_UNDO_ACTION); + EXPECT_EQ(pTrans->failedTimes, 1); + + SNodeMsg rspMsg = {0}; + rspMsg.pNode = pMnode; + int64_t signature = transId; + signature = (signature << 32); + signature += action; + rspMsg.rpcMsg.ahandle = (void *)signature; + rspMsg.rpcMsg.code = 0; + mndTransProcessRsp(&rspMsg); + mndReleaseTrans(pMnode, pTrans); + + pUser2 = mndAcquireUser(pMnode, user2); + ASSERT_NE(pUser2, nullptr); + mndReleaseUser(pMnode, pUser2); + } + + { + transId = 6; + pTrans = mndAcquireTrans(pMnode, transId); + EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR); + EXPECT_EQ(pTrans->stage, TRN_STAGE_UNDO_ACTION); + EXPECT_EQ(pTrans->failedTimes, 2); + + STransAction *pAction = (STransAction *)taosArrayGet(pTrans->undoActions, action); + pAction->msgSent = 1; + + SNodeMsg rspMsg = {0}; + rspMsg.pNode = pMnode; + int64_t signature = transId; + signature = (signature << 32); + signature += action; + rspMsg.rpcMsg.ahandle = (void *)signature; + mndTransProcessRsp(&rspMsg); + mndReleaseTrans(pMnode, pTrans); + + pUser2 = mndAcquireUser(pMnode, user2); + ASSERT_EQ(pUser2, nullptr); + mndReleaseUser(pMnode, pUser2); + } + } } From 1361bf4eedb39601d5b74e7c220150e31e05121b Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 5 May 2022 14:49:00 +0800 Subject: [PATCH 44/48] fix: plan problem of tag scanning --- include/libs/parser/parser.h | 68 +++++++------- source/libs/function/src/builtins.c | 2 +- source/libs/parser/test/mockCatalog.cpp | 22 +++++ source/libs/planner/test/planSTableTest.cpp | 2 + source/libs/planner/test/planTestMain.cpp | 4 +- source/libs/planner/test/planTestUtil.cpp | 99 ++++++++++++++++----- source/libs/planner/test/planTestUtil.h | 2 +- 7 files changed, 142 insertions(+), 57 deletions(-) diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index de71192a9b..84fa166881 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -20,8 +20,8 @@ extern "C" { #endif -#include "querynodes.h" #include "query.h" +#include "querynodes.h" typedef struct SStmtCallback { TAOS_STMT* pStmt; @@ -34,24 +34,26 @@ typedef struct SStmtCallback { typedef struct SParseContext { uint64_t requestId; int32_t acctId; - const char *db; + const char* db; bool topicQuery; - void *pTransporter; + void* pTransporter; SEpSet mgmtEpSet; - const char *pSql; // sql string - size_t sqlLen; // length of the sql string - char *pMsg; // extended error message if exists to help identifying the problem in sql statement. - int32_t msgLen; // max length of the msg - struct SCatalog *pCatalog; - SStmtCallback *pStmtCb; + const char* pSql; // sql string + size_t sqlLen; // length of the sql string + char* pMsg; // extended error message if exists to help identifying the problem in sql statement. + int32_t msgLen; // max length of the msg + struct SCatalog* pCatalog; + SStmtCallback* pStmtCb; + const char* pUser; + bool isSuperUser; } SParseContext; typedef struct SCmdMsgInfo { int16_t msgType; - SEpSet epSet; - void* pMsg; + SEpSet epSet; + void* pMsg; int32_t msgLen; - void* pExtension; // todo remove it soon + void* pExtension; // todo remove it soon } SCmdMsgInfo; typedef enum EQueryExecMode { @@ -63,21 +65,21 @@ typedef enum EQueryExecMode { typedef struct SQuery { EQueryExecMode execMode; - bool haveResultSet; - SNode* pRoot; - int32_t numOfResCols; - SSchema* pResSchema; - int8_t precision; - SCmdMsgInfo* pCmdMsg; - int32_t msgType; - SArray* pDbList; - SArray* pTableList; - bool showRewrite; - int32_t placeholderNum; + bool haveResultSet; + SNode* pRoot; + int32_t numOfResCols; + SSchema* pResSchema; + int8_t precision; + SCmdMsgInfo* pCmdMsg; + int32_t msgType; + SArray* pDbList; + SArray* pTableList; + bool showRewrite; + int32_t placeholderNum; } SQuery; int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery); -bool isInsertSql(const char* pStr, size_t length); +bool isInsertSql(const char* pStr, size_t length); void qDestroyQuery(SQuery* pQueryNode); @@ -89,14 +91,16 @@ int32_t qCloneStmtDataBlock(void** pDst, void* pSrc); void qFreeStmtDataBlock(void* pDataBlock); int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc); void qDestroyStmtDataBlock(void* pBlock); -int32_t qBindStmtColsValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen); -int32_t qBindStmtSingleColValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen, int32_t colIdx, int32_t rowNum); -int32_t qBuildStmtColFields(void *pDataBlock, int32_t *fieldNum, TAOS_FIELD** fields); -int32_t qBuildStmtTagFields(void *pBlock, void *boundTags, int32_t *fieldNum, TAOS_FIELD** fields); -int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, SName *pName, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen); -void destroyBoundColumnInfo(void* pBoundInfo); -int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char *msgBuf, int32_t msgBufLen); - +int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen); +int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx, + int32_t rowNum); +int32_t qBuildStmtColFields(void* pDataBlock, int32_t* fieldNum, TAOS_FIELD** fields); +int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TAOS_FIELD** fields); +int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, SName* pName, TAOS_MULTI_BIND* bind, + char* msgBuf, int32_t msgBufLen); +void destroyBoundColumnInfo(void* pBoundInfo); +int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char* msgBuf, + int32_t msgBufLen); #ifdef __cplusplus } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index bd2fb9dca1..38922833f9 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -913,7 +913,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "tbname", .type = FUNCTION_TYPE_TBNAME, - .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC, + .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_SCAN_PC_FUNC, .translateFunc = translateTbnameColumn, .getEnvFunc = NULL, .initFunc = NULL, diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index ab02ff9558..2d9a907d9b 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -100,6 +100,17 @@ void generateInformationSchema(MockCatalogService* mcs) { } } +/* + * Table:t1 + * Field | Type | DataType | Bytes | + * ========================================================================== + * ts | column | TIMESTAMP | 8 | + * c1 | column | INT | 4 | + * c2 | column | VARCHAR | 20 | + * c3 | column | BIGINT | 8 | + * c4 | column | DOUBLE | 8 | + * c5 | column | DOUBLE | 8 | + */ void generateTestT1(MockCatalogService* mcs) { ITableBuilder& builder = mcs->createTableBuilder("test", "t1", TSDB_NORMAL_TABLE, 6) .setPrecision(TSDB_TIME_PRECISION_MILLI) @@ -113,6 +124,17 @@ void generateTestT1(MockCatalogService* mcs) { builder.done(); } +/* + * Super Table: st1 + * Field | Type | DataType | Bytes | + * ========================================================================== + * ts | column | TIMESTAMP | 8 | + * c1 | column | INT | 4 | + * c2 | column | VARCHAR | 20 | + * tag1 | tag | INT | 4 | + * tag2 | tag | VARCHAR | 20 | + * Child Table: st1s1, st1s2 + */ void generateTestST1(MockCatalogService* mcs) { ITableBuilder& builder = mcs->createTableBuilder("test", "st1", TSDB_SUPER_TABLE, 3, 2) .setPrecision(TSDB_TIME_PRECISION_MILLI) diff --git a/source/libs/planner/test/planSTableTest.cpp b/source/libs/planner/test/planSTableTest.cpp index 0b60eaea32..6e349f8f9c 100644 --- a/source/libs/planner/test/planSTableTest.cpp +++ b/source/libs/planner/test/planSTableTest.cpp @@ -23,4 +23,6 @@ TEST_F(PlanSuperTableTest, tbname) { useDb("root", "test"); run("select tbname from st1"); + + run("select tbname, tag1, tag2 from st1"); } diff --git a/source/libs/planner/test/planTestMain.cpp b/source/libs/planner/test/planTestMain.cpp index a7726973f0..464c636b66 100644 --- a/source/libs/planner/test/planTestMain.cpp +++ b/source/libs/planner/test/planTestMain.cpp @@ -36,11 +36,11 @@ class PlannerEnv : public testing::Environment { static void parseArg(int argc, char* argv[]) { int opt = 0; const char* optstring = ""; - static struct option long_options[] = {{"dump", no_argument, NULL, 'd'}, {0, 0, 0, 0}}; + static struct option long_options[] = {{"dump", optional_argument, NULL, 'd'}, {0, 0, 0, 0}}; while ((opt = getopt_long(argc, argv, optstring, long_options, NULL)) != -1) { switch (opt) { case 'd': - g_isDump = true; + setDumpModule(optarg); break; default: break; diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index 9ddf2a4959..e434a6be96 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -34,7 +34,41 @@ using namespace testing; } \ } while (0); -bool g_isDump = false; +enum DumpModule { + DUMP_MODULE_NOTHING = 1, + DUMP_MODULE_PARSER, + DUMP_MODULE_LOGIC, + DUMP_MODULE_OPTIMIZED, + DUMP_MODULE_SPLIT, + DUMP_MODULE_SCALED, + DUMP_MODULE_PHYSICAL, + DUMP_MODULE_SUBPLAN, + DUMP_MODULE_ALL +}; + +DumpModule g_dumpModule = DUMP_MODULE_NOTHING; + +void setDumpModule(const char* pModule) { + if (NULL == pModule) { + g_dumpModule = DUMP_MODULE_ALL; + } else if (0 == strncasecmp(pModule, "parser", strlen(pModule))) { + g_dumpModule = DUMP_MODULE_PARSER; + } else if (0 == strncasecmp(pModule, "logic", strlen(pModule))) { + g_dumpModule = DUMP_MODULE_LOGIC; + } else if (0 == strncasecmp(pModule, "optimized", strlen(pModule))) { + g_dumpModule = DUMP_MODULE_OPTIMIZED; + } else if (0 == strncasecmp(pModule, "split", strlen(pModule))) { + g_dumpModule = DUMP_MODULE_SPLIT; + } else if (0 == strncasecmp(pModule, "scaled", strlen(pModule))) { + g_dumpModule = DUMP_MODULE_SCALED; + } else if (0 == strncasecmp(pModule, "physical", strlen(pModule))) { + g_dumpModule = DUMP_MODULE_PHYSICAL; + } else if (0 == strncasecmp(pModule, "subplan", strlen(pModule))) { + g_dumpModule = DUMP_MODULE_SUBPLAN; + } else if (0 == strncasecmp(pModule, "all", strlen(pModule))) { + g_dumpModule = DUMP_MODULE_PHYSICAL; + } +} class PlannerTestBaseImpl { public: @@ -66,11 +100,9 @@ class PlannerTestBaseImpl { SQueryPlan* pPlan = nullptr; doCreatePhysiPlan(&cxt, pLogicPlan, &pPlan); - if (g_isDump) { - dump(); - } + dump(g_dumpModule); } catch (...) { - dump(); + dump(DUMP_MODULE_ALL); throw; } } @@ -109,23 +141,48 @@ class PlannerTestBaseImpl { res_.physiSubplans_.clear(); } - void dump() { + void dump(DumpModule module) { + if (DUMP_MODULE_NOTHING == module) { + return; + } + cout << "==========================================sql : [" << stmtEnv_.sql_ << "]" << endl; - cout << "syntax tree : " << endl; - cout << res_.ast_ << endl; - cout << "raw logic plan : " << endl; - cout << res_.rawLogicPlan_ << endl; - cout << "optimized logic plan : " << endl; - cout << res_.optimizedLogicPlan_ << endl; - cout << "split logic plan : " << endl; - cout << res_.splitLogicPlan_ << endl; - cout << "scaled logic plan : " << endl; - cout << res_.scaledLogicPlan_ << endl; - cout << "physical plan : " << endl; - cout << res_.physiPlan_ << endl; - cout << "physical subplan : " << endl; - for (const auto& subplan : res_.physiSubplans_) { - cout << subplan << endl; + + if (DUMP_MODULE_ALL == module || DUMP_MODULE_PARSER == module) { + cout << "syntax tree : " << endl; + cout << res_.ast_ << endl; + } + + if (DUMP_MODULE_ALL == module || DUMP_MODULE_LOGIC == module) { + cout << "raw logic plan : " << endl; + cout << res_.rawLogicPlan_ << endl; + } + + if (DUMP_MODULE_ALL == module || DUMP_MODULE_OPTIMIZED == module) { + cout << "optimized logic plan : " << endl; + cout << res_.optimizedLogicPlan_ << endl; + } + + if (DUMP_MODULE_ALL == module || DUMP_MODULE_SPLIT == module) { + cout << "split logic plan : " << endl; + cout << res_.splitLogicPlan_ << endl; + } + + if (DUMP_MODULE_ALL == module || DUMP_MODULE_SCALED == module) { + cout << "scaled logic plan : " << endl; + cout << res_.scaledLogicPlan_ << endl; + } + + if (DUMP_MODULE_ALL == module || DUMP_MODULE_PHYSICAL == module) { + cout << "physical plan : " << endl; + cout << res_.physiPlan_ << endl; + } + + if (DUMP_MODULE_ALL == module || DUMP_MODULE_SUBPLAN == module) { + cout << "physical subplan : " << endl; + for (const auto& subplan : res_.physiSubplans_) { + cout << subplan << endl; + } } } diff --git a/source/libs/planner/test/planTestUtil.h b/source/libs/planner/test/planTestUtil.h index 7aabff811f..7913ef531f 100644 --- a/source/libs/planner/test/planTestUtil.h +++ b/source/libs/planner/test/planTestUtil.h @@ -32,6 +32,6 @@ class PlannerTestBase : public testing::Test { std::unique_ptr impl_; }; -extern bool g_isDump; +extern void setDumpModule(const char* pModule); #endif // PLAN_TEST_UTIL_H From d457f80afc829cd0ec06a8bdb84588ae660b478a Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 5 May 2022 15:55:11 +0800 Subject: [PATCH 45/48] fix: turn fma off by default (#12114) [TD-15262] --- cmake/cmake.define | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cmake/cmake.define b/cmake/cmake.define index 851d8f9a73..1655154506 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -67,7 +67,13 @@ ELSE () IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64") ADD_DEFINITIONS("-D_TD_ARM_") ELSE () - ADD_DEFINITIONS("-msse4.2 -mfma") + ADD_DEFINITIONS("-msse4.2") + IF("${FMA_SUPPORT}" MATCHES "true") + MESSAGE(STATUS "turn fma function support on") + ADD_DEFINITIONS("-mfma") + ELSE () + MESSAGE(STATUS "turn fma function support off") + ENDIF() ENDIF () ENDIF () From d2f889a3a08e4ea64816ae9ed7020cbb0d126b2c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 5 May 2022 16:13:51 +0800 Subject: [PATCH 46/48] test: add conflict test for transaction --- source/dnode/mnode/impl/inc/mndTrans.h | 1 + source/dnode/mnode/impl/src/mndTrans.c | 6 +- source/dnode/mnode/impl/test/trans/trans2.cpp | 164 ++++++++++++++++-- 3 files changed, 158 insertions(+), 13 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index 2a011f414d..7bd2bd70ee 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -61,6 +61,7 @@ void mndTransSetDbInfo(STrans *pTrans, SDbObj *pDb); int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans); void mndTransProcessRsp(SNodeMsg *pRsp); void mndTransPullup(SMnode *pMnode); +int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 9144de82b3..299667fb45 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1278,7 +1278,7 @@ static int32_t mndProcessTransReq(SNodeMsg *pReq) { return 0; } -static int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans) { +int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans) { SArray *pArray = NULL; if (pTrans->stage == TRN_STAGE_REDO_ACTION) { pArray = pTrans->redoActions; @@ -1296,14 +1296,14 @@ static int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans) { if (pAction == NULL) continue; if (pAction->msgReceived == 0) { - mInfo("trans:%d, action:%d set processed", pTrans->id, i); + mInfo("trans:%d, action:%d set processed for kill msg received", pTrans->id, i); pAction->msgSent = 1; pAction->msgReceived = 1; pAction->errCode = 0; } if (pAction->errCode != 0) { - mInfo("trans:%d, action:%d set processed, errCode from %s to success", pTrans->id, i, + mInfo("trans:%d, action:%d set processed for kill msg received, errCode from %s to success", pTrans->id, i, tstrerror(pAction->errCode)); pAction->msgSent = 1; pAction->msgReceived = 1; diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp index e81827041b..b3cbcb6898 100644 --- a/source/dnode/mnode/impl/test/trans/trans2.cpp +++ b/source/dnode/mnode/impl/test/trans/trans2.cpp @@ -86,7 +86,7 @@ class MndTestTrans2 : public ::testing::Test { void SetUp() override {} void TearDown() override {} - int32_t CreateUserLog(const char *acct, const char *user) { + int32_t CreateUserLog(const char *acct, const char *user, ETrnType type, SDbObj *pDb) { SUserObj userObj = {0}; taosEncryptPass_c((uint8_t *)"taosdata", strlen("taosdata"), userObj.pass); tstrncpy(userObj.user, user, TSDB_USER_LEN); @@ -96,7 +96,7 @@ class MndTestTrans2 : public ::testing::Test { userObj.superUser = 1; SRpcMsg rpcMsg = {0}; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, &rpcMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, type, &rpcMsg); SSdbRaw *pRedoRaw = mndUserActionEncode(&userObj); mndTransAppendRedolog(pTrans, pRedoRaw); sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); @@ -108,13 +108,18 @@ class MndTestTrans2 : public ::testing::Test { char *param = strdup("====> test log <====="); mndTransSetCb(pTrans, TEST_TRANS_START_FUNC, TEST_TRANS_STOP_FUNC, param, strlen(param) + 1); + if (pDb != NULL) { + mndTransSetDbInfo(pTrans, pDb); + } + int32_t code = mndTransPrepare(pMnode, pTrans); mndTransDrop(pTrans); return code; } - int32_t CreateUserAction(const char *acct, const char *user, bool hasUndoAction, ETrnPolicy policy) { + int32_t CreateUserAction(const char *acct, const char *user, bool hasUndoAction, ETrnPolicy policy, ETrnType type, + SDbObj *pDb) { SUserObj userObj = {0}; taosEncryptPass_c((uint8_t *)"taosdata", strlen("taosdata"), userObj.pass); tstrncpy(userObj.user, user, TSDB_USER_LEN); @@ -124,7 +129,7 @@ class MndTestTrans2 : public ::testing::Test { userObj.superUser = 1; SRpcMsg rpcMsg = {0}; - STrans *pTrans = mndTransCreate(pMnode, policy, TRN_TYPE_CREATE_USER, &rpcMsg); + STrans *pTrans = mndTransCreate(pMnode, policy, type, &rpcMsg); SSdbRaw *pRedoRaw = mndUserActionEncode(&userObj); mndTransAppendRedolog(pTrans, pRedoRaw); sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); @@ -176,6 +181,38 @@ class MndTestTrans2 : public ::testing::Test { mndTransSetRpcRsp(pTrans, pRsp, 256); } + if (pDb != NULL) { + mndTransSetDbInfo(pTrans, pDb); + } + + int32_t code = mndTransPrepare(pMnode, pTrans); + mndTransDrop(pTrans); + + return code; + } + + int32_t CreateUserGlobal(const char *acct, const char *user) { + SUserObj userObj = {0}; + taosEncryptPass_c((uint8_t *)"taosdata", strlen("taosdata"), userObj.pass); + tstrncpy(userObj.user, user, TSDB_USER_LEN); + tstrncpy(userObj.acct, acct, TSDB_USER_LEN); + userObj.createdTime = taosGetTimestampMs(); + userObj.updateTime = userObj.createdTime; + userObj.superUser = 1; + + SRpcMsg rpcMsg = {0}; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, &rpcMsg); + SSdbRaw *pRedoRaw = mndUserActionEncode(&userObj); + mndTransAppendRedolog(pTrans, pRedoRaw); + sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); + + SSdbRaw *pUndoRaw = mndUserActionEncode(&userObj); + mndTransAppendUndolog(pTrans, pUndoRaw); + sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED); + + char *param = strdup("====> test log <====="); + mndTransSetCb(pTrans, TEST_TRANS_START_FUNC, TEST_TRANS_STOP_FUNC, param, strlen(param) + 1); + int32_t code = mndTransPrepare(pMnode, pTrans); mndTransDrop(pTrans); @@ -195,12 +232,12 @@ TEST_F(MndTestTrans2, 01_Log) { ASSERT_NE(pMnode, nullptr); - EXPECT_EQ(CreateUserLog(acct, user1), 0); + EXPECT_EQ(CreateUserLog(acct, user1, TRN_TYPE_CREATE_USER, NULL), 0); pUser1 = mndAcquireUser(pMnode, user1); ASSERT_NE(pUser1, nullptr); // failed to create user and rollback - EXPECT_EQ(CreateUserLog(acct_invalid, user2), 0); + EXPECT_EQ(CreateUserLog(acct_invalid, user2, TRN_TYPE_CREATE_USER, NULL), 0); pUser2 = mndAcquireUser(pMnode, user2); ASSERT_EQ(pUser2, nullptr); @@ -222,14 +259,14 @@ TEST_F(MndTestTrans2, 02_Action) { { // failed to create user and rollback - EXPECT_EQ(CreateUserAction(acct, user1, false, TRN_POLICY_ROLLBACK), 0); + EXPECT_EQ(CreateUserAction(acct, user1, false, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, NULL), 0); pUser1 = mndAcquireUser(pMnode, user1); ASSERT_EQ(pUser1, nullptr); mndReleaseUser(pMnode, pUser1); // create user, and fake a response { - EXPECT_EQ(CreateUserAction(acct, user1, true, TRN_POLICY_ROLLBACK), 0); + EXPECT_EQ(CreateUserAction(acct, user1, true, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, NULL), 0); pUser1 = mndAcquireUser(pMnode, user1); ASSERT_NE(pUser1, nullptr); mndReleaseUser(pMnode, pUser1); @@ -259,7 +296,7 @@ TEST_F(MndTestTrans2, 02_Action) { } { - EXPECT_EQ(CreateUserAction(acct, user1, false, TRN_POLICY_RETRY), 0); + EXPECT_EQ(CreateUserAction(acct, user1, false, TRN_POLICY_RETRY, TRN_TYPE_CREATE_USER, NULL), 0); pUser1 = mndAcquireUser(pMnode, user1); ASSERT_NE(pUser1, nullptr); mndReleaseUser(pMnode, pUser1); @@ -315,7 +352,7 @@ TEST_F(MndTestTrans2, 02_Action) { } { - EXPECT_EQ(CreateUserAction(acct, user2, true, TRN_POLICY_ROLLBACK), 0); + EXPECT_EQ(CreateUserAction(acct, user2, true, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, NULL), 0); SUserObj *pUser2 = (SUserObj *)sdbAcquire(pMnode->pSdb, SDB_USER, user2); ASSERT_NE(pUser2, nullptr); mndReleaseUser(pMnode, pUser2); @@ -367,3 +404,110 @@ TEST_F(MndTestTrans2, 02_Action) { } } } + +TEST_F(MndTestTrans2, 03_Kill) { + const char *acct = "root"; + const char *user1 = "kill1"; + const char *user2 = "kill2"; + SUserObj *pUser1 = NULL; + SUserObj *pUser2 = NULL; + STrans *pTrans = NULL; + int32_t transId = 0; + int32_t action = 0; + + ASSERT_NE(pMnode, nullptr); + + { + EXPECT_EQ(CreateUserAction(acct, user1, true, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, NULL), 0); + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_NE(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); + + transId = 7; + pTrans = mndAcquireTrans(pMnode, transId); + EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR); + EXPECT_EQ(pTrans->stage, TRN_STAGE_UNDO_ACTION); + EXPECT_EQ(pTrans->failedTimes, 1); + + mndKillTrans(pMnode, pTrans); + mndReleaseTrans(pMnode, pTrans); + + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_EQ(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); + } +} + +TEST_F(MndTestTrans2, 04_Conflict) { + const char *acct = "root"; + const char *user1 = "conflict1"; + const char *user2 = "conflict2"; + const char *user3 = "conflict3"; + const char *user4 = "conflict4"; + const char *user5 = "conflict5"; + const char *user6 = "conflict6"; + const char *user7 = "conflict7"; + const char *user8 = "conflict8"; + SUserObj *pUser = NULL; + STrans *pTrans = NULL; + int32_t transId = 0; + int32_t code = 0; + + ASSERT_NE(pMnode, nullptr); + + { + SDbObj dbObj1 = {0}; + dbObj1.uid = 9521; + strcpy(dbObj1.name, "db"); + SDbObj dbObj2 = {0}; + dbObj2.uid = 9522; + strcpy(dbObj2.name, "conflict db"); + + EXPECT_EQ(CreateUserAction(acct, user1, true, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_STB, &dbObj1), 0); + pUser = mndAcquireUser(pMnode, user1); + ASSERT_NE(pUser, nullptr); + mndReleaseUser(pMnode, pUser); + + transId = 8; + pTrans = mndAcquireTrans(pMnode, transId); + EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR); + EXPECT_EQ(pTrans->stage, TRN_STAGE_UNDO_ACTION); + + // stb scope + EXPECT_EQ(CreateUserLog(acct, user2, TRN_TYPE_CREATE_DNODE, NULL), -1); + code = terrno; + EXPECT_EQ(code, TSDB_CODE_MND_TRANS_CONFLICT); + + EXPECT_EQ(CreateUserLog(acct, user2, TRN_TYPE_CREATE_DB, &dbObj1), -1); + EXPECT_EQ(CreateUserLog(acct, user2, TRN_TYPE_CREATE_DB, &dbObj2), 0); + EXPECT_EQ(CreateUserLog(acct, user3, TRN_TYPE_CREATE_STB, &dbObj1), 0); + + // db scope + pTrans->type = TRN_TYPE_CREATE_DB; + EXPECT_EQ(CreateUserLog(acct, user4, TRN_TYPE_CREATE_DNODE, NULL), -1); + EXPECT_EQ(CreateUserLog(acct, user4, TRN_TYPE_CREATE_DB, &dbObj1), -1); + EXPECT_EQ(CreateUserLog(acct, user4, TRN_TYPE_CREATE_DB, &dbObj2), 0); + EXPECT_EQ(CreateUserLog(acct, user5, TRN_TYPE_CREATE_STB, &dbObj1), -1); + EXPECT_EQ(CreateUserLog(acct, user5, TRN_TYPE_CREATE_STB, &dbObj2), 0); + + // global scope + pTrans->type = TRN_TYPE_CREATE_DNODE; + EXPECT_EQ(CreateUserLog(acct, user6, TRN_TYPE_CREATE_DNODE, NULL), 0); + EXPECT_EQ(CreateUserLog(acct, user7, TRN_TYPE_CREATE_DB, &dbObj1), -1); + EXPECT_EQ(CreateUserLog(acct, user7, TRN_TYPE_CREATE_DB, &dbObj2), -1); + EXPECT_EQ(CreateUserLog(acct, user7, TRN_TYPE_CREATE_STB, &dbObj1), -1); + EXPECT_EQ(CreateUserLog(acct, user7, TRN_TYPE_CREATE_STB, &dbObj2), -1); + + // global scope + pTrans->type = TRN_TYPE_CREATE_USER; + EXPECT_EQ(CreateUserLog(acct, user7, TRN_TYPE_CREATE_DB, &dbObj1), 0); + EXPECT_EQ(CreateUserLog(acct, user8, TRN_TYPE_CREATE_DB, &dbObj2), 0); + + mndKillTrans(pMnode, pTrans); + mndReleaseTrans(pMnode, pTrans); + + pUser = mndAcquireUser(pMnode, user1); + ASSERT_EQ(pUser, nullptr); + mndReleaseUser(pMnode, pUser); + } +} \ No newline at end of file From 516fca738c5e89ea7ac5aaba8d079b8835d31396 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 5 May 2022 16:31:07 +0800 Subject: [PATCH 47/48] fix: forbid the startup of illegal fqdn --- source/common/src/tglobal.c | 47 +++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 4470f5b0a3..87a9c521af 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -224,7 +224,8 @@ struct SConfig *taosGetCfg() { return tsCfg; } -static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile, char *apolloUrl) { +static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile, + char *apolloUrl) { char cfgDir[PATH_MAX] = {0}; char cfgFile[PATH_MAX + 100] = {0}; @@ -300,15 +301,10 @@ static int32_t taosAddServerLogCfg(SConfig *pCfg) { static int32_t taosAddClientCfg(SConfig *pCfg) { char defaultFqdn[TSDB_FQDN_LEN] = {0}; int32_t defaultServerPort = 6030; - char defaultFirstEp[TSDB_EP_LEN] = {0}; - char defaultSecondEp[TSDB_EP_LEN] = {0}; - if (taosGetFqdn(defaultFqdn) != 0) return -1; - snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort); - snprintf(defaultSecondEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort); - if (cfgAddString(pCfg, "firstEp", defaultFirstEp, 1) != 0) return -1; - if (cfgAddString(pCfg, "secondEp", defaultSecondEp, 1) != 0) return -1; + if (cfgAddString(pCfg, "firstEp", "", 1) != 0) return -1; + if (cfgAddString(pCfg, "secondEp", "", 1) != 0) return -1; if (cfgAddString(pCfg, "fqdn", defaultFqdn, 1) != 0) return -1; if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, 1) != 0) return -1; if (cfgAddDir(pCfg, "tempDir", tsTempDir, 1) != 0) return -1; @@ -478,15 +474,18 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort); + char defaultFirstEp[TSDB_EP_LEN] = {0}; + snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort); + SConfigItem *pFirstEpItem = cfgGetItem(pCfg, "firstEp"); SEp firstEp = {0}; - taosGetFqdnPortFromEp(pFirstEpItem->str, &firstEp); + taosGetFqdnPortFromEp(strlen(pFirstEpItem->str) == 0 ? defaultFirstEp : pFirstEpItem->str, &firstEp); snprintf(tsFirst, sizeof(tsFirst), "%s:%u", firstEp.fqdn, firstEp.port); cfgSetItem(pCfg, "firstEp", tsFirst, pFirstEpItem->stype); SConfigItem *pSecondpItem = cfgGetItem(pCfg, "secondEp"); SEp secondEp = {0}; - taosGetFqdnPortFromEp(pSecondpItem->str, &secondEp); + taosGetFqdnPortFromEp(strlen(pSecondpItem->str) == 0 ? defaultFirstEp : pSecondpItem->str, &secondEp); snprintf(tsSecond, sizeof(tsSecond), "%s:%u", secondEp.fqdn, secondEp.port); cfgSetItem(pCfg, "secondEp", tsSecond, pSecondpItem->stype); @@ -583,8 +582,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { return 0; } -int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, const char *envFile, - char *apolloUrl, SArray *pArgs, bool tsc) { +int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, + const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) { osDefaultInit(); SConfig *pCfg = cfgInit(); @@ -636,7 +635,24 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi return 0; } -int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) { +static int32_t taosCheckGlobalCfg() { + uint32_t ipv4 = taosGetIpv4FromFqdn(tsLocalFqdn); + if (ipv4 == 0xffffffff) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to get ip from fqdn:%s since %s, dnode can not be initialized", tsLocalFqdn, terrstr()); + return -1; + } + + if (tsServerPort <= 0) { + uError("invalid server port:%u, dnode can not be initialized", tsServerPort); + return -1; + } + + return 0; +} + +int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, + bool tsc) { if (tsCfg != NULL) return 0; tsCfg = cfgInit(); @@ -674,6 +690,11 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile taosSetSystemCfg(tsCfg); cfgDumpCfg(tsCfg, tsc, false); + + if (taosCheckGlobalCfg() != 0) { + return -1; + } + return 0; } From 9927b6b014c477ebb04578559dbcc818d644d6ac Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 5 May 2022 17:01:50 +0800 Subject: [PATCH 48/48] fix: invalid write while pullup trans on start --- source/dnode/mnode/impl/src/mndTrans.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 299667fb45..f3438bd0dc 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1000,7 +1000,9 @@ static int32_t mndTransSendActionMsg(SMnode *pMnode, STrans *pTrans, SArray *pAr pAction->msgSent = 0; pAction->msgReceived = 0; pAction->errCode = terrno; - if (terrno == TSDB_CODE_INVALID_PTR) rpcFreeCont(rpcMsg.pCont); + if (terrno == TSDB_CODE_INVALID_PTR || terrno == TSDB_CODE_NODE_OFFLINE) { + rpcFreeCont(rpcMsg.pCont); + } mError("trans:%d, action:%d not send since %s", pTrans->id, action, terrstr()); return -1; }