From d9e832997510a6321aac56bdd5bfef8f59e90513 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 7 Jul 2022 10:05:48 +0800 Subject: [PATCH 01/38] support bool tag --- source/dnode/vnode/src/meta/metaTable.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 74076d323c..b3cdf47302 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -92,7 +92,7 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const } else if (type == TSDB_DATA_TYPE_BOOL) { int val = *(int *)(&pTagVal->i64); int len = sizeof(val); - term = indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_INT, key, nKey, (const char *)&val, len); + term = indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_BOOL, key, nKey, (const char *)&val, len); } if (term != NULL) { indexMultiTermAdd(terms, term); From 11a7b1e1b06e82b41f549cecf38daf900881e633 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 7 Jul 2022 10:18:20 +0800 Subject: [PATCH 02/38] update test case --- tests/system-test/2-query/json_tag.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index ae3f130b7e..24e413fd74 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -323,12 +323,12 @@ class TDTestCase: # where json value is bool tdSql.query("select * from jsons1 where jtag->'tag1'=true") tdSql.checkRows(0) - #tdSql.query("select * from jsons1 where jtag->'tag1'=false") - #tdSql.checkRows(1) + tdSql.query("select * from jsons1 where jtag->'tag1'=false") + tdSql.checkRows(1) tdSql.query("select * from jsons1 where jtag->'tag1'!=false") tdSql.checkRows(0) - #tdSql.query("select * from jsons1 where jtag->'tag1'>false") - #tdSql.checkRows(0) + tdSql.query("select * from jsons1 where jtag->'tag1'>false") + tdSql.checkRows(0) # where json value is null tdSql.query("select * from jsons1 where jtag->'tag1'=null") From 061399e43ed74fd0569f759c469034c77dfdb9db Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 7 Jul 2022 10:56:35 +0800 Subject: [PATCH 03/38] update log --- source/libs/transport/src/transCli.c | 4 ++-- source/libs/transport/src/transSvr.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 8b771f6f8a..4b6ff43ba8 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -573,8 +573,8 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { return; } if (nread < 0) { - tError("%s conn %p read error: %s, ref: %d", CONN_GET_INST_LABEL(conn), conn, uv_err_name(nread), - T_REF_VAL_GET(conn)); + tWarn("%s conn %p read error: %s, ref: %d", CONN_GET_INST_LABEL(conn), conn, uv_err_name(nread), + T_REF_VAL_GET(conn)); conn->broken = true; cliHandleExcept(conn); } diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 3f4a5628f9..b805aa929c 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -305,7 +305,7 @@ void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { return; } - tError("%s conn %p read error: %s", transLabel(pTransInst), conn, uv_err_name(nread)); + tWarn("%s conn %p read error: %s", transLabel(pTransInst), conn, uv_err_name(nread)); if (nread < 0) { conn->broken = true; if (conn->status == ConnAcquire) { From 4d2bc796e74b2c015f955996383bfdf9a9fdd0e0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 7 Jul 2022 11:26:58 +0800 Subject: [PATCH 04/38] enh(query): add block sma for int type column data. --- source/client/test/clientTests.cpp | 23 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 689 ++++--------------------- source/dnode/vnode/src/tsdb/tsdbUtil.c | 20 +- 3 files changed, 134 insertions(+), 598 deletions(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index e8e3237b67..b8e4dcbea0 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -826,9 +826,24 @@ TEST(testCase, update_test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); - taos_query(pConn, "use abc1"); + TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to create database, code:%s", taos_errstr(pRes)); + taos_free_result(pRes); + return; + } - TAOS_RES* pRes = taos_query(pConn, "create table tup (ts timestamp, k int);"); + taos_free_result(pRes); + + pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to use db, code:%s", taos_errstr(pRes)); + taos_free_result(pRes); + return; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table tup (ts timestamp, k int);"); if (taos_errno(pRes) != 0) { printf("failed to create table, reason:%s", taos_errstr(pRes)); } @@ -836,8 +851,8 @@ TEST(testCase, update_test) { taos_free_result(pRes); char s[256] = {0}; - for(int32_t i = 0; i < 7000; ++i) { - sprintf(s, "insert into tup values('2020-1-1 1:1:1', %d)", i); + for(int32_t i = 0; i < 17000; ++i) { + sprintf(s, "insert into tup values(now+%da, %d)", i, i); pRes = taos_query(pConn, s); taos_free_result(pRes); } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index ae9caa3444..b7429a5b89 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -48,7 +48,7 @@ typedef struct SBlockOrderSupporter { typedef struct SIOCostSummary { int64_t blockLoadTime; - int64_t statisInfoLoadTime; + int64_t smaLoadTime; int64_t checkForNextTime; int64_t headFileLoad; int64_t headFileLoadTime; @@ -63,10 +63,10 @@ typedef struct SBlockLoadSuppInfo { } SBlockLoadSuppInfo; typedef struct SFilesetIter { - int32_t numOfFiles; // number of total files - int32_t index; // current accessed index in the list - SArray* pFileList; // data file list - int32_t order; + int32_t numOfFiles; // number of total files + int32_t index; // current accessed index in the list + SArray* pFileList; // data file list + int32_t order; } SFilesetIter; typedef struct SFileDataBlockInfo { @@ -122,20 +122,6 @@ struct STsdbReader { STSchema* pSchema; SDataFReader* pFileReader; SVersionRange verRange; -#if 0 - SArray* prev; // previous row which is before than time window - SArray* next; // next row which is after the query time window - SFileBlockInfo* pDataBlockInfo; - SDataCols* pDataCols; // in order to hold current file data block - int32_t allocSize; // allocated data block size - SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */ - SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQueryAttr */ - // SDFileSet* pFileGroup; - // SFSIter fileIter; - // SReadH rhelper; - // SColumnDataAgg* statis; // query level statistics, only one table block statistics info exists at any time - // SColumnDataAgg** pstatis;// the ptr array list to return to caller -#endif }; static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter); @@ -247,33 +233,6 @@ static STimeWindow updateQueryTimeWindow(STsdb* pTsdb, STimeWindow* pWindow) { return win; } -// todo remove this -static void setQueryTimewindow(STsdbReader* pReader, SQueryTableDataCond* pCond, int32_t tWinIdx) { - // pReader->window = pCond->twindows[tWinIdx]; - - // bool updateTs = false; - // int64_t startTs = updateQueryTimeWindow(pReader->pTsdb); - // if (ASCENDING_TRAVERSE(pReader->order)) { - // if (startTs > pReader->window.skey) { - // pReader->window.skey = startTs; - // pCond->twindows[tWinIdx].skey = startTs; - // updateTs = true; - // } - // } else { - // if (startTs > pReader->window.ekey) { - // pReader->window.ekey = startTs; - // pCond->twindows[tWinIdx].ekey = startTs; - // updateTs = true; - // } - // } - - // if (updateTs) { - // tsdbDebug("%p update the query time window, old:%" PRId64 " - %" PRId64 ", new:%" PRId64 " - %" PRId64 ", %s", - // pReader, pCond->twindows[tWinIdx].skey, pCond->twindows[tWinIdx].ekey, pReader->window.skey, - // pReader->window.ekey, pReader->idStr); - // } -} - static void limitOutputBufferSize(const SQueryTableDataCond* pCond, int32_t* capacity) { int32_t rowLen = 0; for (int32_t i = 0; i < pCond->numOfCols; ++i) { @@ -399,8 +358,6 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd pReader->type = pCond->type; pReader->window = updateQueryTimeWindow(pVnode->pTsdb, pCond->twindows); - // todo remove this - setQueryTimewindow(pReader, pCond, 0); ASSERT(pCond->numOfCols > 0); limitOutputBufferSize(pCond, &pReader->capacity); @@ -914,200 +871,6 @@ _error: // return midPos; // } -// static int32_t mergeTwoRowFromMem(STsdbReader* pTsdbReadHandle, int32_t capacity, int32_t* curRow, STSRow* row1, -// STSRow* row2, int32_t numOfCols, uint64_t uid, STSchema* pSchema1, STSchema* -// pSchema2, bool update, TSKEY* lastRowKey) { -// #if 1 -// STSchema* pSchema; -// STSRow* row; -// int16_t colId; -// int16_t offset; - -// bool isRow1DataRow = TD_IS_TP_ROW(row1); -// bool isRow2DataRow; -// bool isChosenRowDataRow; -// int32_t chosen_itr; -// SCellVal sVal = {0}; -// TSKEY rowKey = TSKEY_INITIAL_VAL; -// int32_t nResult = 0; -// int32_t mergeOption = 0; // 0 discard 1 overwrite 2 merge - -// // the schema version info is embeded in STSRow -// int32_t numOfColsOfRow1 = 0; - -// if (pSchema1 == NULL) { -// pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row1)); -// } - -// #ifdef TD_DEBUG_PRINT_ROW -// char flags[70] = {0}; -// STsdb* pTsdb = pTsdbReadHandle->rhelper.pRepo; -// snprintf(flags, 70, "%s:%d vgId:%d dir:%s row1%s=NULL,row2%s=NULL", __func__, __LINE__, TD_VID(pTsdb->pVnode), -// pTsdb->dir, row1 ? "!" : "", row2 ? "!" : ""); -// tdSRowPrint(row1, pSchema1, flags); -// #endif - -// if (isRow1DataRow) { -// numOfColsOfRow1 = schemaNCols(pSchema1); -// } else { -// numOfColsOfRow1 = tdRowGetNCols(row1); -// } - -// int32_t numOfColsOfRow2 = 0; -// if (row2) { -// isRow2DataRow = TD_IS_TP_ROW(row2); -// if (pSchema2 == NULL) { -// pSchema2 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row2)); -// } -// if (isRow2DataRow) { -// numOfColsOfRow2 = schemaNCols(pSchema2); -// } else { -// numOfColsOfRow2 = tdRowGetNCols(row2); -// } -// } - -// int32_t i = 0, j = 0, k = 0; -// while (i < numOfCols && (j < numOfColsOfRow1 || k < numOfColsOfRow2)) { -// SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); - -// int32_t colIdOfRow1; -// if (j >= numOfColsOfRow1) { -// colIdOfRow1 = INT32_MAX; -// } else if (isRow1DataRow) { -// colIdOfRow1 = pSchema1->columns[j].colId; -// } else { -// colIdOfRow1 = tdKvRowColIdAt(row1, j); -// } - -// int32_t colIdOfRow2; -// if (k >= numOfColsOfRow2) { -// colIdOfRow2 = INT32_MAX; -// } else if (isRow2DataRow) { -// colIdOfRow2 = pSchema2->columns[k].colId; -// } else { -// colIdOfRow2 = tdKvRowColIdAt(row2, k); -// } - -// if (colIdOfRow1 < colIdOfRow2) { // the most probability -// if (colIdOfRow1 < pColInfo->info.colId) { -// ++j; -// continue; -// } -// row = row1; -// pSchema = pSchema1; -// isChosenRowDataRow = isRow1DataRow; -// chosen_itr = j; -// } else if (colIdOfRow1 == colIdOfRow2) { -// if (colIdOfRow1 < pColInfo->info.colId) { -// ++j; -// ++k; -// continue; -// } -// row = row1; -// pSchema = pSchema1; -// isChosenRowDataRow = isRow1DataRow; -// chosen_itr = j; -// } else { -// if (colIdOfRow2 < pColInfo->info.colId) { -// ++k; -// continue; -// } -// row = row2; -// pSchema = pSchema2; -// chosen_itr = k; -// isChosenRowDataRow = isRow2DataRow; -// } - -// if (isChosenRowDataRow) { -// colId = pSchema->columns[chosen_itr].colId; -// offset = pSchema->columns[chosen_itr].offset; -// // TODO: use STSRowIter -// tdSTpRowGetVal(row, colId, pSchema->columns[chosen_itr].type, pSchema->flen, offset, chosen_itr - 1, &sVal); -// if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { -// rowKey = *(TSKEY*)sVal.val; -// if (rowKey != *lastRowKey) { -// mergeOption = 1; -// if (*lastRowKey != TSKEY_INITIAL_VAL) { -// ++(*curRow); -// } -// *lastRowKey = rowKey; -// ++nResult; -// } else if (update) { -// mergeOption = 2; -// } else { -// mergeOption = 0; -// break; -// } -// } -// } else { -// // TODO: use STSRowIter -// if (chosen_itr == 0) { -// colId = PRIMARYKEY_TIMESTAMP_COL_ID; -// tdSKvRowGetVal(row, PRIMARYKEY_TIMESTAMP_COL_ID, -1, -1, &sVal); -// rowKey = *(TSKEY*)sVal.val; -// if (rowKey != *lastRowKey) { -// mergeOption = 1; -// if (*lastRowKey != TSKEY_INITIAL_VAL) { -// ++(*curRow); -// } -// *lastRowKey = rowKey; -// ++nResult; -// } else if (update) { -// mergeOption = 2; -// } else { -// mergeOption = 0; -// break; -// } -// } else { -// SKvRowIdx* pColIdx = tdKvRowColIdxAt(row, chosen_itr - 1); -// colId = pColIdx->colId; -// offset = pColIdx->offset; -// tdSKvRowGetVal(row, colId, offset, chosen_itr - 1, &sVal); -// } -// } - -// ASSERT(rowKey != TSKEY_INITIAL_VAL); - -// if (colId == pColInfo->info.colId) { -// if (tdValTypeIsNorm(sVal.valType)) { -// colDataAppend(pColInfo, *curRow, sVal.val, false); -// } else if (tdValTypeIsNull(sVal.valType)) { -// colDataAppend(pColInfo, *curRow, NULL, true); -// } else if (tdValTypeIsNone(sVal.valType)) { -// // TODO: Set null if nothing append for this row -// if (mergeOption == 1) { -// colDataAppend(pColInfo, *curRow, NULL, true); -// } -// } else { -// ASSERT(0); -// } - -// ++i; - -// if (row == row1) { -// ++j; -// } else { -// ++k; -// } -// } else { -// if (mergeOption == 1) { -// colDataAppend(pColInfo, *curRow, NULL, true); -// } -// ++i; -// } -// } - -// if (mergeOption == 1) { -// while (i < numOfCols) { // the remain columns are all null data -// SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); -// colDataAppend(pColInfo, *curRow, NULL, true); -// ++i; -// } -// } - -// return nResult; -// #endif -// } // static void doCheckGeneratedBlockRange(STsdbReader* pTsdbReadHandle) { // SQueryFilePos* cur = &pTsdbReadHandle->cur; @@ -1387,66 +1150,6 @@ _error: // pTsdbReadHandle->idStr); // } -// int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { -// int firstPos, lastPos, midPos = -1; -// int numOfRows; -// TSKEY* keyList; - -// if (num <= 0) return -1; - -// keyList = (TSKEY*)pValue; -// firstPos = 0; -// lastPos = num - 1; - -// if (order == TSDB_ORDER_DESC) { -// // find the first position which is smaller than the key -// while (1) { -// if (key >= keyList[lastPos]) return lastPos; -// if (key == keyList[firstPos]) return firstPos; -// if (key < keyList[firstPos]) return firstPos - 1; - -// numOfRows = lastPos - firstPos + 1; -// midPos = (numOfRows >> 1) + firstPos; - -// if (key < keyList[midPos]) { -// lastPos = midPos - 1; -// } else if (key > keyList[midPos]) { -// firstPos = midPos + 1; -// } else { -// break; -// } -// } - -// } else { -// // find the first position which is bigger than the key -// while (1) { -// if (key <= keyList[firstPos]) return firstPos; -// if (key == keyList[lastPos]) return lastPos; - -// if (key > keyList[lastPos]) { -// lastPos = lastPos + 1; -// if (lastPos >= num) -// return -1; -// else -// return lastPos; -// } - -// numOfRows = lastPos - firstPos + 1; -// midPos = (numOfRows >> 1) + firstPos; - -// if (key < keyList[midPos]) { -// lastPos = midPos - 1; -// } else if (key > keyList[midPos]) { -// firstPos = midPos + 1; -// } else { -// break; -// } -// } -// } - -// return midPos; -// } - static void cleanupBlockOrderSupporter(SBlockOrderSupporter* pSup) { taosMemoryFreeClear(pSup->numOfBlocksPerTable); taosMemoryFreeClear(pSup->indexPerTable); @@ -2882,162 +2585,6 @@ int32_t tsdbGetStbIdList(SMeta* pMeta, int64_t suid, SArray* list) { return TSDB_CODE_SUCCESS; } -// static void destroyHelper(void* param) { -// if (param == NULL) { -// return; -// } - -// // tQueryInfo* pInfo = (tQueryInfo*)param; -// // if (pInfo->optr != TSDB_RELATION_IN) { -// // taosMemoryFreeClear(pInfo->q); -// // } else { -// // taosHashCleanup((SHashObj *)(pInfo->q)); -// // } - -// taosMemoryFree(param); -// } - -// #define TSDB_PREV_ROW 0x1 -// #define TSDB_NEXT_ROW 0x2 - -// static bool loadBlockOfActiveTable(STsdbReader* pTsdbReadHandle) { -// if (pTsdbReadHandle->checkFiles) { -// // check if the query range overlaps with the file data block -// bool exists = true; - -// int32_t code = buildBlockFromFiles(pTsdbReadHandle, &exists); -// if (code != TSDB_CODE_SUCCESS) { -// pTsdbReadHandle->checkFiles = false; -// return false; -// } - -// if (exists) { -// tsdbRetrieveDataBlock((STsdbReader**)pTsdbReadHandle, NULL); -// if (pTsdbReadHandle->currentLoadExternalRows && pTsdbReadHandle->window.skey == pTsdbReadHandle->window.ekey) { -// SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, 0); -// assert(*(int64_t*)pColInfo->pData == pTsdbReadHandle->window.skey); -// } - -// pTsdbReadHandle->currentLoadExternalRows = false; // clear the flag, since the exact matched row is found. -// return exists; -// } - -// pTsdbReadHandle->checkFiles = false; -// } - -// if (hasMoreDataInCache(pTsdbReadHandle)) { -// pTsdbReadHandle->currentLoadExternalRows = false; -// return true; -// } - -// // current result is empty -// if (pTsdbReadHandle->currentLoadExternalRows && pTsdbReadHandle->window.skey == pTsdbReadHandle->window.ekey && -// pTsdbReadHandle->cur.rows == 0) { -// // SMemTable* pMemRef = pTsdbReadHandle->pMemTable; - -// // doGetExternalRow(pTsdbReadHandle, TSDB_PREV_ROW, pMemRef); -// // doGetExternalRow(pTsdbReadHandle, TSDB_NEXT_ROW, pMemRef); - -// bool result = tsdbGetExternalRow(pTsdbReadHandle); - -// // pTsdbReadHandle->prev = doFreeColumnInfoData(pTsdbReadHandle->prev); -// // pTsdbReadHandle->next = doFreeColumnInfoData(pTsdbReadHandle->next); -// pTsdbReadHandle->currentLoadExternalRows = false; - -// return result; -// } - -// return false; -// } - -// static bool loadDataBlockFromTableSeq(STsdbReader* pTsdbReadHandle) { -// size_t numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); -// assert(numOfTables > 0); - -// int64_t stime = taosGetTimestampUs(); - -// while (pTsdbReadHandle->activeIndex < numOfTables) { -// if (loadBlockOfActiveTable(pTsdbReadHandle)) { -// return true; -// } - -// STableBlockScanInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, pTsdbReadHandle->activeIndex); -// pCheckInfo->numOfBlocks = 0; - -// pTsdbReadHandle->activeIndex += 1; -// pTsdbReadHandle->locateStart = false; -// pTsdbReadHandle->checkFiles = true; -// pTsdbReadHandle->cur.rows = 0; -// pTsdbReadHandle->currentLoadExternalRows = pTsdbReadHandle->loadExternalRow; - -// terrno = TSDB_CODE_SUCCESS; - -// int64_t elapsedTime = taosGetTimestampUs() - stime; -// pTsdbReadHandle->cost.checkForNextTime += elapsedTime; -// } - -// return false; -// } - -// bool tsdbGetExternalRow(STsdbReader* pHandle) { -// STsdbReader* pTsdbReadHandle = (STsdbReader*)pHandle; -// SQueryFilePos* cur = &pTsdbReadHandle->cur; - -// cur->fid = INT32_MIN; -// cur->mixBlock = true; -// if (pTsdbReadHandle->prev == NULL || pTsdbReadHandle->next == NULL) { -// cur->rows = 0; -// return false; -// } - -// int32_t numOfCols = (int32_t)QH_GET_NUM_OF_COLS(pTsdbReadHandle); -// for (int32_t i = 0; i < numOfCols; ++i) { -// SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, i); -// SColumnInfoData* first = taosArrayGet(pTsdbReadHandle->prev, i); - -// memcpy(pColInfoData->pData, first->pData, pColInfoData->info.bytes); - -// SColumnInfoData* sec = taosArrayGet(pTsdbReadHandle->next, i); -// memcpy(((char*)pColInfoData->pData) + pColInfoData->info.bytes, sec->pData, pColInfoData->info.bytes); - -// if (i == 0 && pColInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP) { -// cur->win.skey = *(TSKEY*)pColInfoData->pData; -// cur->win.ekey = *(TSKEY*)(((char*)pColInfoData->pData) + TSDB_KEYSIZE); -// } -// } - -// cur->rows = 2; -// return true; -// } - -// static void* doFreeColumnInfoData(SArray* pColumnInfoData) { -// if (pColumnInfoData == NULL) { -// return NULL; -// } - -// size_t cols = taosArrayGetSize(pColumnInfoData); -// for (int32_t i = 0; i < cols; ++i) { -// SColumnInfoData* pColInfo = taosArrayGet(pColumnInfoData, i); -// colDataDestroy(pColInfo); -// } - -// taosArrayDestroy(pColumnInfoData); -// return NULL; -// } - -// static void* destroyTableCheckInfo(SArray* pTableCheckInfo) { -// size_t size = taosArrayGetSize(pTableCheckInfo); -// for (int32_t i = 0; i < size; ++i) { -// STableBlockScanInfo* p = taosArrayGet(pTableCheckInfo, i); -// destroyTableMemIterator(p); - -// taosMemoryFreeClear(p->pCompInfo); -// } - -// taosArrayDestroy(pTableCheckInfo); -// return NULL; -// } - // ====================================== EXPOSED APIs ====================================== int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTableList, STsdbReader** ppReader, const char* idstr) { @@ -3128,7 +2675,7 @@ void tsdbReaderClose(STsdbReader* pReader) { tsdbDebug("%p :io-cost summary: head-file read cnt:%" PRIu64 ", head-file time:%" PRIu64 " us, statis-info:%" PRId64 " us, datablock:%" PRId64 " us, check data:%" PRId64 " us, %s", - pReader, pCost->headFileLoad, pCost->headFileLoadTime, pCost->statisInfoLoadTime, pCost->blockLoadTime, + pReader, pCost->headFileLoad, pCost->headFileLoadTime, pCost->smaLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pReader->idStr); taosMemoryFree(pReader->idStr); @@ -3185,31 +2732,31 @@ int32_t tsdbRetrieveDataBlockStatisInfo(STsdbReader* pReader, SColumnDataAgg*** int32_t code = 0; *allHave = false; + // there is no statistics data for composed block if (pReader->status.composedDataBlock) { *pBlockStatis = NULL; return TSDB_CODE_SUCCESS; } - // SFileBlockInfo* pBlockInfo = &pReader->pDataBlockInfo[c->slot]; - // assert((c->slot >= 0 && c->slot < pReader->numOfBlocks) || ((c->slot == pReader->numOfBlocks) && (c->slot == 0))); + SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(&pReader->status.blockIter); + STableBlockScanInfo* pBlockScanInfo = taosHashGet(pReader->status.pTableMap, &pFBlock->uid, sizeof(pFBlock->uid)); + SBlock* pBlock = taosArrayGet(pBlockScanInfo->pBlockList, pFBlock->tbBlockIdx); - // // file block with sub-blocks has no statistics data - // if (pBlockInfo->compBlock->numOfSubBlocks > 1) { - // *pBlockStatis = NULL; - // return TSDB_CODE_SUCCESS; - // } + int64_t stime = taosGetTimestampUs(); - // int64_t stime = taosGetTimestampUs(); - // int statisStatus = tsdbLoadBlockStatis(&pReader->rhelper, pBlockInfo->compBlock); - // if (statisStatus < TSDB_STATIS_OK) { - // return terrno; - // } else if (statisStatus > TSDB_STATIS_OK) { - // *pBlockStatis = NULL; - // return TSDB_CODE_SUCCESS; - // } + if (tBlockHasSma(pBlock)) { + SArray* pColAgg = taosArrayInit(4, sizeof(SColumnDataAgg)); + code = tsdbReadBlockSma(pReader->pFileReader, pBlock, pColAgg, NULL); + if (code != TSDB_CODE_SUCCESS) { + tsdbDebug("vgId:%d, failed to load block SMA for uid %" PRIu64", code:%s, %s", 0, pFBlock->uid, + tstrerror(code), pReader->idStr); + return code; + } + } - // tsdbDebug("vgId:%d, succeed to load block statis part for uid %" PRIu64, REPO_ID(pReader->pTsdb), - // TSDB_READ_TABLE_UID(&pReader->rhelper)); + int64_t el = taosGetTimestampUs() - stime; + tsdbDebug("vgId:%d, succeed to load block SMA for uid %" PRIu64", elapsed time:%"PRId64"us, %s", 0, pFBlock->uid, + el, pReader->idStr); // int16_t* colIds = pReader->suppInfo.defaultLoadColumn->pData; @@ -3224,34 +2771,36 @@ int32_t tsdbRetrieveDataBlockStatisInfo(STsdbReader* pReader, SColumnDataAgg*** // *allHave = true; // tsdbGetBlockStatis(&pReader->rhelper, pReader->suppInfo.pstatis, (int)numOfCols, pBlockInfo->compBlock); - // // always load the first primary timestamp column data - // SColumnDataAgg* pPrimaryColStatis = &pReader->suppInfo.pstatis[0]; - // assert(pPrimaryColStatis->colId == PRIMARYKEY_TIMESTAMP_COL_ID); + // always load the first primary timestamp column data + SColumnDataAgg* pTsAgg = &pReader->suppInfo.pstatis[0]; + assert(pTsAgg->colId == PRIMARYKEY_TIMESTAMP_COL_ID); - // pPrimaryColStatis->numOfNull = 0; - // pPrimaryColStatis->min = pBlockInfo->compBlock->minKey.ts; - // pPrimaryColStatis->max = pBlockInfo->compBlock->maxKey.ts; - // pReader->suppInfo.plist[0] = &pReader->suppInfo.pstatis[0]; + pTsAgg->numOfNull = 0; + pTsAgg->min = pReader->pResBlock->info.window.skey; + pTsAgg->max = pReader->pResBlock->info.window.ekey; + pReader->suppInfo.plist[0] = &pReader->suppInfo.pstatis[0]; - // // update the number of NULL data rows - // int32_t* slotIds = pReader->suppInfo.slotIds; - // for (int32_t i = 1; i < numOfCols; ++i) { - // ASSERT(colIds[i] == pReader->pSchema->columns[slotIds[i]].colId); - // if (IS_BSMA_ON(&(pReader->pSchema->columns[slotIds[i]]))) { - // if (pReader->suppInfo.pstatis[i].numOfNull == -1) { // set the column data are all NULL - // pReader->suppInfo.pstatis[i].numOfNull = pBlockInfo->compBlock->numOfRows; - // } + // update the number of NULL data rows + size_t numOfCols = blockDataGetNumOfCols(pReader->pResBlock); + int32_t* slotIds = pReader->suppInfo.slotIds; - // pReader->suppInfo.plist[i] = &pReader->suppInfo.pstatis[i]; - // } else { - // *allHave = false; - // } - // } + for (int32_t i = 1; i < numOfCols; ++i) { +// ASSERT(colIds[i] == pReader->pSchema->columns[slotIds[i]].colId); + if (IS_BSMA_ON(&(pReader->pSchema->columns[slotIds[i]]))) { + if (pReader->suppInfo.pstatis[i].numOfNull == -1) { // set the column data are all NULL +// pReader->suppInfo.pstatis[i].numOfNull = pBlockInfo->compBlock->numOfRows; + } - // int64_t elapsed = taosGetTimestampUs() - stime; - // pReader->cost.statisInfoLoadTime += elapsed; + pReader->suppInfo.plist[i] = &pReader->suppInfo.pstatis[i]; + } else { + *allHave = false; + } + } - // *pBlockStatis = pReader->suppInfo.plist; + int64_t elapsed = taosGetTimestampUs() - stime; + pReader->cost.smaLoadTime += elapsed; + + *pBlockStatis = pReader->suppInfo.plist; return code; } @@ -3286,8 +2835,6 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond, int32_ return TSDB_CODE_SUCCESS; } - setQueryTimewindow(pReader, pCond, tWinIdx); - pReader->order = pCond->order; pReader->type = BLOCK_LOAD_OFFSET_ORDER; pReader->status.loadFromFile = true; @@ -3324,114 +2871,72 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond, int32_ return code; } +static int32_t getBucketIndex(int32_t startRow, int32_t bucketRange, int32_t numOfRows) { + return (numOfRows - startRow) / bucketRange; +} + int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTableBlockInfo) { - int32_t code = 0; - // pTableBlockInfo->totalSize = 0; - // pTableBlockInfo->totalRows = 0; + int32_t code = TSDB_CODE_SUCCESS; + pTableBlockInfo->totalSize = 0; + pTableBlockInfo->totalRows = 0; - // STsdbFS* pFileHandle = REPO_FS(pReader->pTsdb); + // find the start data block in file + SReaderStatus* pStatus = &pReader->status; - // // find the start data block in file - // pReader->locateStart = true; - // STsdbKeepCfg* pCfg = REPO_KEEP_CFG(pReader->pTsdb); - // int32_t fid = getFileIdFromKey(pReader->window.skey, pCfg->days, pCfg->precision); + STsdbCfg* pc = &pReader->pTsdb->pVnode->config.tsdbCfg; + pTableBlockInfo->defMinRows = pc->minRows; + pTableBlockInfo->defMaxRows = pc->maxRows; - // tsdbRLockFS(pFileHandle); - // tsdbFSIterInit(&pReader->fileIter, pFileHandle, pReader->order); - // tsdbFSIterSeek(&pReader->fileIter, fid); - // tsdbUnLockFS(pFileHandle); + int32_t bucketRange = ceil((pc->maxRows - pc->minRows) / 20.0); - // STsdbCfg* pc = REPO_CFG(pReader->pTsdb); - // pTableBlockInfo->defMinRows = pc->minRows; - // pTableBlockInfo->defMaxRows = pc->maxRows; + pTableBlockInfo->numOfFiles += 1; - // int32_t bucketRange = ceil((pc->maxRows - pc->minRows) / 20.0); + int32_t numOfTables = (int32_t)taosHashGetSize(pStatus->pTableMap); + int defaultRows = 4096; - // pTableBlockInfo->numOfFiles += 1; + SDataBlockIter* pBlockIter = &pStatus->blockIter; + pTableBlockInfo->numOfFiles += pStatus->fileIter.numOfFiles; + pTableBlockInfo->numOfBlocks += pBlockIter->numOfBlocks; - // int32_t code = TSDB_CODE_SUCCESS; - // int32_t numOfBlocks = 0; - // int32_t numOfTables = (int32_t)taosArrayGetSize(pReader->pTableCheckInfo); - // int defaultRows = 4096; - // STimeWindow win = TSWINDOW_INITIALIZER; + pTableBlockInfo->numOfTables = numOfTables; - // while (true) { - // numOfBlocks = 0; - // tsdbRLockFS(REPO_FS(pReader->pTsdb)); + while (true) { + bool hasNext = blockIteratorNext(&pStatus->blockIter); + if (hasNext) { + SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(pBlockIter); + STableBlockScanInfo* pScanInfo = taosHashGet(pStatus->pTableMap, &pFBlock->uid, sizeof(pFBlock->uid)); + SBlock* pBlock = taosArrayGet(pScanInfo->pBlockList, pFBlock->tbBlockIdx); - // if ((pReader->pFileGroup = tsdbFSIterNext(&pReader->fileIter)) == NULL) { - // tsdbUnLockFS(REPO_FS(pReader->pTsdb)); - // break; - // } + int32_t numOfRows = pBlock->nRow; + pTableBlockInfo->totalRows += numOfRows; - // tsdbGetFidKeyRange(pCfg->days, pCfg->precision, pReader->pFileGroup->fid, &win.skey, &win.ekey); + if (numOfRows > pTableBlockInfo->maxRows) { + pTableBlockInfo->maxRows = numOfRows; + } - // // current file are not overlapped with query time window, ignore remain files - // if ((win.skey > pReader->window.ekey) /* || (!ascTraverse && win.ekey < pTsdbReadHandle->window.ekey)*/) { - // tsdbUnLockFS(REPO_FS(pReader->pTsdb)); - // tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, %s", pReader, - // pReader->window.skey, pReader->window.ekey, pReader->idStr); - // pReader->pFileGroup = NULL; - // break; - // } + if (numOfRows < pTableBlockInfo->minRows) { + pTableBlockInfo->minRows = numOfRows; + } - // pTableBlockInfo->numOfFiles += 1; - // if (tsdbSetAndOpenReadFSet(&pReader->rhelper, pReader->pFileGroup) < 0) { - // tsdbUnLockFS(REPO_FS(pReader->pTsdb)); - // code = terrno; - // break; - // } + if (numOfRows < defaultRows) { + pTableBlockInfo->numOfSmallBlocks += 1; + } - // tsdbUnLockFS(REPO_FS(pReader->pTsdb)); + int32_t bucketIndex = getBucketIndex(pTableBlockInfo->defMinRows, bucketRange, numOfRows); + pTableBlockInfo->blockRowsHisto[bucketIndex]++; + } else { + code = initForFirstBlockInFile(pReader, pBlockIter); + if ((code != TSDB_CODE_SUCCESS) || (pReader->status.loadFromFile == false)) { + break; + } - // if (tsdbLoadBlockIdx(&pReader->rhelper) < 0) { - // code = terrno; - // break; - // } + pTableBlockInfo->numOfBlocks += pBlockIter->numOfBlocks; + } - // if ((code = getFileCompInfo(pReader, &numOfBlocks)) != TSDB_CODE_SUCCESS) { - // break; - // } +// tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %s", pReader, numOfBlocks, numOfTables, +// pReader->pFileGroup->fid, pReader->idStr); + } - // tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %s", pReader, numOfBlocks, numOfTables, - // pReader->pFileGroup->fid, pReader->idStr); - - // if (numOfBlocks == 0) { - // continue; - // } - - // pTableBlockInfo->numOfBlocks += numOfBlocks; - - // for (int32_t i = 0; i < numOfTables; ++i) { - // STableBlockScanInfo* pCheckInfo = taosArrayGet(pReader->pTableCheckInfo, i); - - // SBlock* pBlock = pCheckInfo->pCompInfo->blocks; - - // for (int32_t j = 0; j < pCheckInfo->numOfBlocks; ++j) { - // pTableBlockInfo->totalSize += pBlock[j].len; - - // int32_t numOfRows = pBlock[j].numOfRows; - // pTableBlockInfo->totalRows += numOfRows; - - // if (numOfRows > pTableBlockInfo->maxRows) { - // pTableBlockInfo->maxRows = numOfRows; - // } - - // if (numOfRows < pTableBlockInfo->minRows) { - // pTableBlockInfo->minRows = numOfRows; - // } - - // if (numOfRows < defaultRows) { - // pTableBlockInfo->numOfSmallBlocks += 1; - // } - - // int32_t bucketIndex = getBucketIndex(pTableBlockInfo->defMinRows, bucketRange, numOfRows); - // pTableBlockInfo->blockRowsHisto[bucketIndex]++; - // } - // } - // } - - // pTableBlockInfo->numOfTables = numOfTables; return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index f906ef1b54..25c8598696 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1230,10 +1230,26 @@ void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) { break; case TSDB_DATA_TYPE_SMALLINT: break; - case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_INT: { + pColAgg->sum += colVal.value.i32; + if (pColAgg->min > colVal.value.i32) { + pColAgg->min = colVal.value.i32; + } + if (pColAgg->max < colVal.value.i32) { + pColAgg->max = colVal.value.i32; + } break; - case TSDB_DATA_TYPE_BIGINT: + } + case TSDB_DATA_TYPE_BIGINT: { + pColAgg->sum += colVal.value.i64; + if (pColAgg->min > colVal.value.i64) { + pColAgg->min = colVal.value.i64; + } + if (pColAgg->max < colVal.value.i64) { + pColAgg->max = colVal.value.i64; + } break; + } case TSDB_DATA_TYPE_FLOAT: break; case TSDB_DATA_TYPE_DOUBLE: From bd693eaa0287893b798abedbd732541d9b37cc04 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Thu, 7 Jul 2022 11:42:01 +0800 Subject: [PATCH 05/38] test: let case run about and or for byte compute --- tests/system-test/2-query/and_or_for_byte.py | 4 ++-- tests/system-test/fulltest.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/system-test/2-query/and_or_for_byte.py b/tests/system-test/2-query/and_or_for_byte.py index 28d3e1cf43..78ba0e8dfa 100644 --- a/tests/system-test/2-query/and_or_for_byte.py +++ b/tests/system-test/2-query/and_or_for_byte.py @@ -230,7 +230,7 @@ class TDTestCase: work_sql += f"cast({arg} as bigint){opera}" if not agg: - work_sql+= f" from {tbname} order by ts" + work_sql+= f" from {tbname} order by tbname ,ts" else: work_sql+= f" from {tbname} " tdSql.query(work_sql) @@ -243,7 +243,7 @@ class TDTestCase: else: origin_sql += f"cast({arg} as bigint)," if not agg: - origin_sql+= f" from {tbname} order by ts" + origin_sql+= f" from {tbname} order by tbname ,ts" else: origin_sql+= f" from {tbname} " tdSql.query(origin_sql) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 0b91b556cc..67d2065d2c 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -117,7 +117,7 @@ python3 ./test.py -f 2-query/distribute_agg_avg.py python3 ./test.py -f 2-query/distribute_agg_stddev.py python3 ./test.py -f 2-query/twa.py python3 ./test.py -f 2-query/irate.py -#python3 ./test.py -f 2-query/and_or_for_byte.py +python3 ./test.py -f 2-query/and_or_for_byte.py python3 ./test.py -f 2-query/function_null.py python3 ./test.py -f 2-query/queryQnode.py From 9a0e15f1c182b7526c6a7ad633ae6f862b60ab72 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 7 Jul 2022 05:32:24 +0000 Subject: [PATCH 06/38] fix coredump --- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index c22d1a4064..38bc5f855e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1178,7 +1178,7 @@ int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnD } // check - if (!taosCheckChecksumWhole(NULL, size)) { + if (!taosCheckChecksumWhole(*ppBuf, size)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; } From d9e5172357f546591d5abb1e5c4958d930d42685 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 7 Jul 2022 15:32:56 +0800 Subject: [PATCH 07/38] fix(query): set the correct sma data --- source/client/test/clientTests.cpp | 9 +-- source/dnode/vnode/src/tsdb/tsdbRead.c | 79 +++++++++++++------------ source/libs/function/src/builtinsimpl.c | 31 ++++------ 3 files changed, 56 insertions(+), 63 deletions(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index b8e4dcbea0..69cca1441a 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -750,7 +750,7 @@ TEST(testCase, projection_query_stables) { taos_close(pConn); } - +#endif TEST(testCase, agg_query_tables) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); @@ -763,7 +763,7 @@ TEST(testCase, agg_query_tables) { } taos_free_result(pRes); - pRes = taos_query(pConn, "show table distributed st1"); + pRes = taos_query(pConn, "show table distributed tup"); if (taos_errno(pRes) != 0) { printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); taos_free_result(pRes); @@ -775,6 +775,7 @@ TEST(testCase, agg_query_tables) { taos_close(pConn); } +#if 0 /* --- copy the following script in the shell to setup the environment --- @@ -820,7 +821,7 @@ TEST(testCase, async_api_test) { getchar(); taos_close(pConn); } -#endif + TEST(testCase, update_test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -857,5 +858,5 @@ TEST(testCase, update_test) { taos_free_result(pRes); } } - +#endif #pragma GCC diagnostic pop diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index b7429a5b89..a1ac165797 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -55,7 +55,7 @@ typedef struct SIOCostSummary { } SIOCostSummary; typedef struct SBlockLoadSuppInfo { - SColumnDataAgg* pstatis; + SColumnDataAgg tsColAgg; SColumnDataAgg** plist; int16_t* colIds; // column ids for loading file block data int32_t* slotIds; // colId to slotId @@ -364,13 +364,14 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd // allocate buffer in order to load data blocks from file SBlockLoadSuppInfo* pSup = &pReader->suppInfo; - pSup->pstatis = taosMemoryCalloc(pCond->numOfCols, sizeof(SColumnDataAgg)); pSup->plist = taosMemoryCalloc(pCond->numOfCols, POINTER_BYTES); - if (pSup->pstatis == NULL || pSup->plist == NULL) { + if (pSup->plist == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _end; } + pSup->tsColAgg.colId = PRIMARYKEY_TIMESTAMP_COL_ID; + pReader->pResBlock = createResBlock(pCond, pReader->capacity); if (pReader->pResBlock == NULL) { code = terrno; @@ -2647,8 +2648,6 @@ void tsdbReaderClose(STsdbReader* pReader) { } blockDataDestroy(pReader->pResBlock); - - taosMemoryFreeClear(pReader->suppInfo.pstatis); taosMemoryFreeClear(pReader->suppInfo.plist); taosMemoryFree(pReader->suppInfo.slotIds); @@ -2744,56 +2743,48 @@ int32_t tsdbRetrieveDataBlockStatisInfo(STsdbReader* pReader, SColumnDataAgg*** int64_t stime = taosGetTimestampUs(); + SArray* pColAgg = taosArrayInit(4, sizeof(SColumnDataAgg)); if (tBlockHasSma(pBlock)) { - SArray* pColAgg = taosArrayInit(4, sizeof(SColumnDataAgg)); code = tsdbReadBlockSma(pReader->pFileReader, pBlock, pColAgg, NULL); if (code != TSDB_CODE_SUCCESS) { tsdbDebug("vgId:%d, failed to load block SMA for uid %" PRIu64", code:%s, %s", 0, pFBlock->uid, tstrerror(code), pReader->idStr); return code; } + } else { + *pBlockStatis = NULL; + return TSDB_CODE_SUCCESS; } - int64_t el = taosGetTimestampUs() - stime; - tsdbDebug("vgId:%d, succeed to load block SMA for uid %" PRIu64", elapsed time:%"PRId64"us, %s", 0, pFBlock->uid, - el, pReader->idStr); - - // int16_t* colIds = pReader->suppInfo.defaultLoadColumn->pData; - - // size_t numOfCols = QH_GET_NUM_OF_COLS(pReader); - // memset(pReader->suppInfo.plist, 0, numOfCols * POINTER_BYTES); - // memset(pReader->suppInfo.pstatis, 0, numOfCols * sizeof(SColumnDataAgg)); - - // for (int32_t i = 0; i < numOfCols; ++i) { - // pReader->suppInfo.pstatis[i].colId = colIds[i]; - // } - - // *allHave = true; - // tsdbGetBlockStatis(&pReader->rhelper, pReader->suppInfo.pstatis, (int)numOfCols, pBlockInfo->compBlock); + *allHave = true; // always load the first primary timestamp column data - SColumnDataAgg* pTsAgg = &pReader->suppInfo.pstatis[0]; - assert(pTsAgg->colId == PRIMARYKEY_TIMESTAMP_COL_ID); + SColumnDataAgg* pTsAgg = &pReader->suppInfo.tsColAgg; - pTsAgg->numOfNull = 0; + pTsAgg->numOfNull = 0; + pTsAgg->colId = PRIMARYKEY_TIMESTAMP_COL_ID; pTsAgg->min = pReader->pResBlock->info.window.skey; pTsAgg->max = pReader->pResBlock->info.window.ekey; - pReader->suppInfo.plist[0] = &pReader->suppInfo.pstatis[0]; + pReader->suppInfo.plist[0] = pTsAgg; // update the number of NULL data rows size_t numOfCols = blockDataGetNumOfCols(pReader->pResBlock); - int32_t* slotIds = pReader->suppInfo.slotIds; - for (int32_t i = 1; i < numOfCols; ++i) { -// ASSERT(colIds[i] == pReader->pSchema->columns[slotIds[i]].colId); - if (IS_BSMA_ON(&(pReader->pSchema->columns[slotIds[i]]))) { - if (pReader->suppInfo.pstatis[i].numOfNull == -1) { // set the column data are all NULL -// pReader->suppInfo.pstatis[i].numOfNull = pBlockInfo->compBlock->numOfRows; + int32_t i = 0, j = 0; + while(j < numOfCols && i < taosArrayGetSize(pColAgg)) { + SColumnDataAgg* pAgg = taosArrayGet(pColAgg, i); + if (pAgg->colId == pReader->suppInfo.colIds[j]) { + if (IS_BSMA_ON(&(pReader->pSchema->columns[i]))) { + pReader->suppInfo.plist[j] = pAgg; + i += 1; + j += 1; + } else { + *allHave = false; } - - pReader->suppInfo.plist[i] = &pReader->suppInfo.pstatis[i]; - } else { - *allHave = false; + } else if (pAgg->colId < pReader->suppInfo.colIds[j]) { + i += 1; + } else if (pReader->suppInfo.colIds[j] < pAgg->colId) { + j += 1; } } @@ -2801,6 +2792,10 @@ int32_t tsdbRetrieveDataBlockStatisInfo(STsdbReader* pReader, SColumnDataAgg*** pReader->cost.smaLoadTime += elapsed; *pBlockStatis = pReader->suppInfo.plist; + + tsdbDebug("vgId:%d, succeed to load block SMA for uid %" PRIu64", elapsed time:%"PRId64"us, %s", 0, pFBlock->uid, + elapsed, pReader->idStr); + return code; } @@ -2843,7 +2838,7 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond, int32_ pReader->window = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows[tWinIdx]); // allocate buffer in order to load data blocks from file - memset(pReader->suppInfo.pstatis, 0, sizeof(SColumnDataAgg)); + memset(&pReader->suppInfo.tsColAgg, 0, sizeof(SColumnDataAgg)); memset(pReader->suppInfo.plist, 0, POINTER_BYTES); // todo set the correct numOfTables @@ -2899,9 +2894,9 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa pTableBlockInfo->numOfBlocks += pBlockIter->numOfBlocks; pTableBlockInfo->numOfTables = numOfTables; + bool hasNext = true; while (true) { - bool hasNext = blockIteratorNext(&pStatus->blockIter); if (hasNext) { SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(pBlockIter); STableBlockScanInfo* pScanInfo = taosHashGet(pStatus->pTableMap, &pFBlock->uid, sizeof(pFBlock->uid)); @@ -2924,6 +2919,9 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa int32_t bucketIndex = getBucketIndex(pTableBlockInfo->defMinRows, bucketRange, numOfRows); pTableBlockInfo->blockRowsHisto[bucketIndex]++; + + hasNext = blockIteratorNext(&pStatus->blockIter); + } else { code = initForFirstBlockInFile(pReader, pBlockIter); if ((code != TSDB_CODE_SUCCESS) || (pReader->status.loadFromFile == false)) { @@ -2933,6 +2931,11 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa pTableBlockInfo->numOfBlocks += pBlockIter->numOfBlocks; } +/* + hasNext = blockIteratorNext(&pStatus->blockIter); +*/ + + // tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %s", pReader, numOfBlocks, numOfTables, // pReader->pFileGroup->fid, pReader->idStr); } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index f94522f0d8..03532caac0 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -5547,30 +5547,18 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { } } - int32_t delta = maxVal - minVal; - int32_t step = delta / 50; - if (step == 0) { - step = 1; - } + // maximum number of step is 80 + double factor = pData->numOfBlocks / 80.0; int32_t numOfBuckets = sizeof(pData->blockRowsHisto) / sizeof(pData->blockRowsHisto[0]); - int32_t bucketRange = (pData->maxRows - pData->minRows) / numOfBuckets; - - bool singleModel = false; - if (bucketRange == 0) { - singleModel = true; - step = 20; - bucketRange = (pData->defMaxRows - pData->defMinRows) / numOfBuckets; - } + int32_t bucketRange = (pData->defMaxRows - pData->defMinRows) / numOfBuckets; for (int32_t i = 0; i < tListLen(pData->blockRowsHisto); ++i) { - len = sprintf(st + VARSTR_HEADER_SIZE, "%04d |", pData->defMinRows + bucketRange * (i + 1)); + len = sprintf(st + VARSTR_HEADER_SIZE, "%04d |", pData->defMinRows + bucketRange * i); int32_t num = 0; - if (singleModel && pData->blockRowsHisto[i] > 0) { - num = 20; - } else { - num = (pData->blockRowsHisto[i] + step - 1) / step; + if (pData->blockRowsHisto[i] > 0) { + num = (pData->blockRowsHisto[i]) / factor; } for (int32_t j = 0; j < num; ++j) { @@ -5578,9 +5566,10 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { len += x; } - double v = pData->blockRowsHisto[i] * 100.0 / pData->numOfBlocks; - len += sprintf(st + VARSTR_HEADER_SIZE + len, " %d (%.2f%c)", pData->blockRowsHisto[i], v, '%'); - printf("%s\n", st); + if (num > 0) { + double v = pData->blockRowsHisto[i] * 100.0 / pData->numOfBlocks; + len += sprintf(st + VARSTR_HEADER_SIZE + len, " %d (%.2f%c)", pData->blockRowsHisto[i], v, '%'); + } varDataSetLen(st, len); colDataAppend(pColInfo, row++, st, false); From 44d7397e18a4869bbf043a92a793960f8be8df1a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 7 Jul 2022 15:33:53 +0800 Subject: [PATCH 08/38] test:update the unit test. --- source/client/test/clientTests.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 69cca1441a..78a446eaad 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -750,7 +750,6 @@ TEST(testCase, projection_query_stables) { taos_close(pConn); } -#endif TEST(testCase, agg_query_tables) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); @@ -775,7 +774,6 @@ TEST(testCase, agg_query_tables) { taos_close(pConn); } -#if 0 /* --- copy the following script in the shell to setup the environment --- @@ -821,6 +819,7 @@ TEST(testCase, async_api_test) { getchar(); taos_close(pConn); } +#endif TEST(testCase, update_test) { From 6ac3ce0213fea80e19f11b262b68a48108e40823 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 7 Jul 2022 15:34:59 +0800 Subject: [PATCH 09/38] fix(query): fix syntax error. --- source/client/test/clientTests.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 78a446eaad..7927c1e008 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -857,5 +857,4 @@ TEST(testCase, update_test) { taos_free_result(pRes); } } -#endif #pragma GCC diagnostic pop From aab82fc0809d28e38683ef89397a4a99b990480a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 7 Jul 2022 15:38:17 +0800 Subject: [PATCH 10/38] fix(query): check return code while loading sma --- source/libs/executor/src/scanoperator.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b3a00cd6f2..8b8d6903c1 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -210,7 +210,10 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca bool allColumnsHaveAgg = true; SColumnDataAgg** pColAgg = NULL; - tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->dataReader, &pColAgg, &allColumnsHaveAgg); + int32_t code = tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->dataReader, &pColAgg, &allColumnsHaveAgg); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } if (allColumnsHaveAgg == true) { int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); From 57bf509dee14cbe55dd0f3f036f88369c24fde02 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 7 Jul 2022 15:40:04 +0800 Subject: [PATCH 11/38] refactor: do some internal refactor. --- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- source/libs/executor/src/executorimpl.c | 4 ++-- source/libs/executor/src/scanoperator.c | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 6320f4719d..60a731a9a8 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -131,7 +131,7 @@ int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, SArray *pTabl void tsdbReaderClose(STsdbReader *pReader); bool tsdbNextDataBlock(STsdbReader *pReader); void tsdbRetrieveDataBlockInfo(STsdbReader *pReader, SDataBlockInfo *pDataBlockInfo); -int32_t tsdbRetrieveDataBlockStatisInfo(STsdbReader *pReader, SColumnDataAgg ***pBlockStatis, bool *allHave); +int32_t tsdbRetrieveDatablockSMAInfo(STsdbReader *pReader, SColumnDataAgg ***pBlockStatis, bool *allHave); SArray *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond, int32_t tWinIdx); int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index a1ac165797..db8c348673 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2727,7 +2727,7 @@ void tsdbRetrieveDataBlockInfo(STsdbReader* pReader, SDataBlockInfo* pDataBlockI pDataBlockInfo->window = pReader->pResBlock->info.window; } -int32_t tsdbRetrieveDataBlockStatisInfo(STsdbReader* pReader, SColumnDataAgg*** pBlockStatis, bool* allHave) { +int32_t tsdbRetrieveDatablockSMAInfo(STsdbReader* pReader, SColumnDataAgg*** pBlockStatis, bool* allHave) { int32_t code = 0; *allHave = false; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 52c8960102..1e72aee3e3 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1130,7 +1130,7 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc } else if ((*status) == BLK_DATA_SMA_LOAD) { // this function never returns error? pCost->loadBlockStatis += 1; -// tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockAgg); +// tsdbRetrieveDatablockSMAInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockAgg); if (pBlock->pBlockAgg == NULL) { // data block statistics does not exist, load data block // pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL); @@ -1141,7 +1141,7 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc // load the data block statistics to perform further filter pCost->loadBlockStatis += 1; -// tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockAgg); +// tsdbRetrieveDatablockSMAInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockAgg); if (pQueryAttr->topBotQuery && pBlock->pBlockAgg != NULL) { { // set previous window diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8b8d6903c1..eb7bdfebab 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -210,7 +210,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca bool allColumnsHaveAgg = true; SColumnDataAgg** pColAgg = NULL; - int32_t code = tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->dataReader, &pColAgg, &allColumnsHaveAgg); + int32_t code = tsdbRetrieveDatablockSMAInfo(pTableScanInfo->dataReader, &pColAgg, &allColumnsHaveAgg); if (code != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, code); } @@ -2230,7 +2230,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc bool allColumnsHaveAgg = true; SColumnDataAgg** pColAgg = NULL; STsdbReader* reader = taosArrayGetP(pTableScanInfo->dataReaders, readerIdx); - tsdbRetrieveDataBlockStatisInfo(reader, &pColAgg, &allColumnsHaveAgg); + tsdbRetrieveDatablockSMAInfo(reader, &pColAgg, &allColumnsHaveAgg); if (allColumnsHaveAgg == true) { int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); From 41be3e4d0468dbcdcc4c19054664eb589bfd807c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 7 Jul 2022 15:56:43 +0800 Subject: [PATCH 12/38] refactor: do some internal refactor. --- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 82 +++++++++++++------------ source/libs/executor/src/executorimpl.c | 4 +- source/libs/executor/src/scanoperator.c | 4 +- 4 files changed, 47 insertions(+), 45 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 60a731a9a8..59ee89762e 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -131,7 +131,7 @@ int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, SArray *pTabl void tsdbReaderClose(STsdbReader *pReader); bool tsdbNextDataBlock(STsdbReader *pReader); void tsdbRetrieveDataBlockInfo(STsdbReader *pReader, SDataBlockInfo *pDataBlockInfo); -int32_t tsdbRetrieveDatablockSMAInfo(STsdbReader *pReader, SColumnDataAgg ***pBlockStatis, bool *allHave); +int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SColumnDataAgg ***pBlockStatis, bool *allHave); SArray *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond, int32_t tWinIdx); int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index db8c348673..d64fc15143 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -55,6 +55,7 @@ typedef struct SIOCostSummary { } SIOCostSummary; typedef struct SBlockLoadSuppInfo { + SArray* pColAgg; SColumnDataAgg tsColAgg; SColumnDataAgg** plist; int16_t* colIds; // column ids for loading file block data @@ -364,8 +365,9 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd // allocate buffer in order to load data blocks from file SBlockLoadSuppInfo* pSup = &pReader->suppInfo; + pSup->pColAgg = taosArrayInit(4, sizeof(SColumnDataAgg)); pSup->plist = taosMemoryCalloc(pCond->numOfCols, POINTER_BYTES); - if (pSup->plist == NULL) { + if (pSup->pColAgg == NULL || pSup->plist == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _end; } @@ -2649,13 +2651,10 @@ void tsdbReaderClose(STsdbReader* pReader) { blockDataDestroy(pReader->pResBlock); taosMemoryFreeClear(pReader->suppInfo.plist); + + taosArrayDestroy(pReader->suppInfo.pColAgg); taosMemoryFree(pReader->suppInfo.slotIds); - if (!isEmptyQueryTimeWindow(&pReader->window)) { - // tsdbMayUnTakeMemSnapshot(pTsdbReadHandle); - } else { - ASSERT(pReader->status.pTableMap == NULL); - } #if 0 // if (pReader->status.pTableScanInfo != NULL) { // pReader->status.pTableScanInfo = destroyTableCheckInfo(pReader->status.pTableScanInfo); @@ -2727,7 +2726,7 @@ void tsdbRetrieveDataBlockInfo(STsdbReader* pReader, SDataBlockInfo* pDataBlockI pDataBlockInfo->window = pReader->pResBlock->info.window; } -int32_t tsdbRetrieveDatablockSMAInfo(STsdbReader* pReader, SColumnDataAgg*** pBlockStatis, bool* allHave) { +int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockStatis, bool* allHave) { int32_t code = 0; *allHave = false; @@ -2743,12 +2742,13 @@ int32_t tsdbRetrieveDatablockSMAInfo(STsdbReader* pReader, SColumnDataAgg*** pBl int64_t stime = taosGetTimestampUs(); - SArray* pColAgg = taosArrayInit(4, sizeof(SColumnDataAgg)); + SBlockLoadSuppInfo* pSup = &pReader->suppInfo; + if (tBlockHasSma(pBlock)) { - code = tsdbReadBlockSma(pReader->pFileReader, pBlock, pColAgg, NULL); + code = tsdbReadBlockSma(pReader->pFileReader, pBlock, pSup->pColAgg, NULL); if (code != TSDB_CODE_SUCCESS) { - tsdbDebug("vgId:%d, failed to load block SMA for uid %" PRIu64", code:%s, %s", 0, pFBlock->uid, - tstrerror(code), pReader->idStr); + tsdbDebug("vgId:%d, failed to load block SMA for uid %" PRIu64 ", code:%s, %s", 0, pFBlock->uid, tstrerror(code), + pReader->idStr); return code; } } else { @@ -2756,44 +2756,44 @@ int32_t tsdbRetrieveDatablockSMAInfo(STsdbReader* pReader, SColumnDataAgg*** pBl return TSDB_CODE_SUCCESS; } - *allHave = true; + *allHave = true; - // always load the first primary timestamp column data - SColumnDataAgg* pTsAgg = &pReader->suppInfo.tsColAgg; + // always load the first primary timestamp column data + SColumnDataAgg* pTsAgg = &pSup->tsColAgg; pTsAgg->numOfNull = 0; pTsAgg->colId = PRIMARYKEY_TIMESTAMP_COL_ID; - pTsAgg->min = pReader->pResBlock->info.window.skey; - pTsAgg->max = pReader->pResBlock->info.window.ekey; - pReader->suppInfo.plist[0] = pTsAgg; + pTsAgg->min = pReader->pResBlock->info.window.skey; + pTsAgg->max = pReader->pResBlock->info.window.ekey; + pSup->plist[0] = pTsAgg; - // update the number of NULL data rows - size_t numOfCols = blockDataGetNumOfCols(pReader->pResBlock); + // update the number of NULL data rows + size_t numOfCols = blockDataGetNumOfCols(pReader->pResBlock); - int32_t i = 0, j = 0; - while(j < numOfCols && i < taosArrayGetSize(pColAgg)) { - SColumnDataAgg* pAgg = taosArrayGet(pColAgg, i); - if (pAgg->colId == pReader->suppInfo.colIds[j]) { - if (IS_BSMA_ON(&(pReader->pSchema->columns[i]))) { - pReader->suppInfo.plist[j] = pAgg; - i += 1; - j += 1; - } else { - *allHave = false; - } - } else if (pAgg->colId < pReader->suppInfo.colIds[j]) { - i += 1; - } else if (pReader->suppInfo.colIds[j] < pAgg->colId) { - j += 1; - } - } + int32_t i = 0, j = 0; + while (j < numOfCols && i < taosArrayGetSize(pSup->pColAgg)) { + SColumnDataAgg* pAgg = taosArrayGet(pSup->pColAgg, i); + if (pAgg->colId == pSup->colIds[j]) { + if (IS_BSMA_ON(&(pReader->pSchema->columns[i]))) { + pSup->plist[j] = pAgg; + i += 1; + j += 1; + } else { + *allHave = false; + } + } else if (pAgg->colId < pSup->colIds[j]) { + i += 1; + } else if (pSup->colIds[j] < pAgg->colId) { + j += 1; + } + } - int64_t elapsed = taosGetTimestampUs() - stime; - pReader->cost.smaLoadTime += elapsed; + int64_t elapsed = taosGetTimestampUs() - stime; + pReader->cost.smaLoadTime += elapsed; - *pBlockStatis = pReader->suppInfo.plist; + *pBlockStatis = pSup->plist; - tsdbDebug("vgId:%d, succeed to load block SMA for uid %" PRIu64", elapsed time:%"PRId64"us, %s", 0, pFBlock->uid, + tsdbDebug("vgId:%d, succeed to load block SMA for uid %" PRIu64 ", elapsed time:%" PRId64 "us, %s", 0, pFBlock->uid, elapsed, pReader->idStr); return code; @@ -2841,6 +2841,8 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond, int32_ memset(&pReader->suppInfo.tsColAgg, 0, sizeof(SColumnDataAgg)); memset(pReader->suppInfo.plist, 0, POINTER_BYTES); + pReader->suppInfo.tsColAgg.colId = PRIMARYKEY_TIMESTAMP_COL_ID; + // todo set the correct numOfTables int32_t numOfTables = 1; SDataBlockIter* pBlockIter = &pReader->status.blockIter; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 1e72aee3e3..ee6b58c3fb 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1130,7 +1130,7 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc } else if ((*status) == BLK_DATA_SMA_LOAD) { // this function never returns error? pCost->loadBlockStatis += 1; -// tsdbRetrieveDatablockSMAInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockAgg); +// tsdbRetrieveDatablockSMA(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockAgg); if (pBlock->pBlockAgg == NULL) { // data block statistics does not exist, load data block // pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL); @@ -1141,7 +1141,7 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc // load the data block statistics to perform further filter pCost->loadBlockStatis += 1; -// tsdbRetrieveDatablockSMAInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockAgg); +// tsdbRetrieveDatablockSMA(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockAgg); if (pQueryAttr->topBotQuery && pBlock->pBlockAgg != NULL) { { // set previous window diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index eb7bdfebab..c6b4938ac7 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -210,7 +210,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca bool allColumnsHaveAgg = true; SColumnDataAgg** pColAgg = NULL; - int32_t code = tsdbRetrieveDatablockSMAInfo(pTableScanInfo->dataReader, &pColAgg, &allColumnsHaveAgg); + int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, &pColAgg, &allColumnsHaveAgg); if (code != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, code); } @@ -2230,7 +2230,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc bool allColumnsHaveAgg = true; SColumnDataAgg** pColAgg = NULL; STsdbReader* reader = taosArrayGetP(pTableScanInfo->dataReaders, readerIdx); - tsdbRetrieveDatablockSMAInfo(reader, &pColAgg, &allColumnsHaveAgg); + tsdbRetrieveDatablockSMA(reader, &pColAgg, &allColumnsHaveAgg); if (allColumnsHaveAgg == true) { int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); From dd4f67df2fe61ddf9f5c2e86e6df3f70b3487f42 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 7 Jul 2022 08:01:01 +0000 Subject: [PATCH 13/38] fix: sma read bug --- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 38bc5f855e..0cf56621aa 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1152,7 +1152,7 @@ _err: int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg, uint8_t **ppBuf) { int32_t code = 0; TdFilePtr pFD = pReader->pSmaFD; - int64_t offset = pBlock->aSubBlock[0].offset; + int64_t offset = pBlock->aSubBlock[0].sOffset; int64_t size = pBlock->aSubBlock[0].nSma * sizeof(SColumnDataAgg) + sizeof(TSCKSUM); uint8_t *pBuf = NULL; int64_t n; @@ -1175,6 +1175,9 @@ int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnD if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; + } else if (n < size) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _err; } // check From 1c3c270307cc9df5abbc9ed68f27605c6de3596f Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 7 Jul 2022 17:00:01 +0800 Subject: [PATCH 14/38] fix(sync): snapshot strategy wal first --- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 2 + source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 2 + source/libs/sync/inc/syncInt.h | 1 - source/libs/sync/src/syncAppendEntries.c | 17 +++---- source/libs/sync/src/syncAppendEntriesReply.c | 48 +++++++++++++++++-- source/libs/sync/src/syncMain.c | 19 ++++---- source/libs/sync/src/syncMessage.c | 2 +- source/libs/sync/src/syncRaftCfg.c | 8 ++-- source/libs/sync/src/syncReplication.c | 19 +++++++- tests/script/tsim/sync/vnodesnapshot.sim | 4 +- 10 files changed, 91 insertions(+), 31 deletions(-) diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 59d68b2110..f4ff64520e 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -237,10 +237,12 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_BATCH, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_BATCH, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_SEND, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 1f7347203d..c81f77fa56 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -376,10 +376,12 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_BATCH, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_BATCH, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_SET_VNODE_STANDBY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 8936cd6ed9..9742f93824 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -67,7 +67,6 @@ typedef struct SSyncNode { char path[TSDB_FILENAME_LEN]; char raftStorePath[TSDB_FILENAME_LEN * 2]; char configPath[TSDB_FILENAME_LEN * 2]; - int32_t batchSize; // sync io SWal* pWal; diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 885ab1acae..2d511b6003 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -842,8 +842,8 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc do { char logBuf[128]; snprintf(logBuf, sizeof(logBuf), - "recv sync-append-entries-batch, fake match2, pre-index:%ld, pre-term:%lu, datalen:%d", - pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen); + "recv sync-append-entries-batch, fake match2, {pre-index:%ld, pre-term:%lu, datalen:%d, datacount:%d}", + pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen, pMsg->dataCount); syncNodeEventLog(ths, logBuf); } while (0); @@ -876,7 +876,7 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc code = syncNodePreCommit(ths, pAppendEntry); ASSERT(code == 0); - syncEntryDestory(pAppendEntry); + // syncEntryDestory(pAppendEntry); } // fsync once @@ -931,8 +931,8 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc do { char logBuf[128]; snprintf(logBuf, sizeof(logBuf), - "recv sync-append-entries-batch, not match, pre-index:%ld, pre-term:%lu, datalen:%d", - pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen); + "recv sync-append-entries-batch, not match, {pre-index:%ld, pre-term:%lu, datalen:%d, datacount:%d}", + pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen, pMsg->dataCount); syncNodeEventLog(ths, logBuf); } while (0); @@ -976,8 +976,9 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc do { char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, match, pre-index:%ld, pre-term:%lu, datalen:%d", - pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen); + snprintf(logBuf, sizeof(logBuf), + "recv sync-append-entries, match, {pre-index:%ld, pre-term:%lu, datalen:%d, datacount:%d}", + pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen, pMsg->dataCount); syncNodeEventLog(ths, logBuf); } while (0); @@ -999,7 +1000,7 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc code = syncNodePreCommit(ths, pAppendEntry); ASSERT(code == 0); - syncEntryDestory(pAppendEntry); + // syncEntryDestory(pAppendEntry); } // fsync once diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index e4724fc999..3f19b6759e 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -174,8 +174,12 @@ int32_t syncNodeOnAppendEntriesReplySnapshot2Cb(SSyncNode* ths, SyncAppendEntrie SyncIndex newNextIndex = pMsg->matchIndex + 1; SyncIndex newMatchIndex = pMsg->matchIndex; - if (ths->pLogStore->syncLogExist(ths->pLogStore, newNextIndex) && - ths->pLogStore->syncLogExist(ths->pLogStore, newNextIndex - 1)) { + bool needStartSnapshot = false; + if (newMatchIndex >= SYNC_INDEX_BEGIN && !ths->pLogStore->syncLogExist(ths->pLogStore, newMatchIndex)) { + needStartSnapshot = true; + } + + if (!needStartSnapshot) { // update next-index, match-index syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), newNextIndex); syncIndexMgrSetIndex(ths->pMatchIndex, &(pMsg->srcId), newMatchIndex); @@ -197,15 +201,36 @@ int32_t syncNodeOnAppendEntriesReplySnapshot2Cb(SSyncNode* ths, SyncAppendEntrie syncIndexMgrSetIndex(ths->pMatchIndex, &(pMsg->srcId), newMatchIndex); } + // event log, update next-index + do { + char host[64]; + int16_t port; + syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); + + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "reset next-index:%ld, match-index:%ld for %s:%d", newNextIndex, newMatchIndex, + host, port); + syncNodeEventLog(ths, logBuf); + + } while (0); + } else { SyncIndex nextIndex = syncIndexMgrGetIndex(ths->pNextIndex, &(pMsg->srcId)); if (nextIndex > SYNC_INDEX_BEGIN) { --nextIndex; - if (ths->pLogStore->syncLogExist(ths->pLogStore, nextIndex) && - ths->pLogStore->syncLogExist(ths->pLogStore, nextIndex - 1)) { + bool needStartSnapshot = false; + if (nextIndex >= SYNC_INDEX_BEGIN && !ths->pLogStore->syncLogExist(ths->pLogStore, nextIndex)) { + needStartSnapshot = true; + } + if (nextIndex - 1 >= SYNC_INDEX_BEGIN && !ths->pLogStore->syncLogExist(ths->pLogStore, nextIndex - 1)) { + needStartSnapshot = true; + } + + if (!needStartSnapshot) { // do nothing + } else { SSyncRaftEntry* pEntry; int32_t code = ths->pLogStore->syncLogGetEntry(ths->pLogStore, nextIndex, &pEntry); @@ -227,6 +252,21 @@ int32_t syncNodeOnAppendEntriesReplySnapshot2Cb(SSyncNode* ths, SyncAppendEntrie nextIndex = SYNC_INDEX_BEGIN; } syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), nextIndex); + + // event log, update next-index + do { + char host[64]; + int16_t port; + syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); + + SyncIndex newNextIndex = nextIndex; + SyncIndex newMatchIndex = syncIndexMgrGetIndex(ths->pMatchIndex, &(pMsg->srcId)); + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "reset2 next-index:%ld, match-index:%ld for %s:%d", newNextIndex, newMatchIndex, + host, port); + syncNodeEventLog(ths, logBuf); + + } while (0); } return 0; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index e207d89212..5951078822 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -648,8 +648,6 @@ void setHeartbeatTimerMS(int64_t rid, int32_t hbTimerMS) { } int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak) { - int32_t ret = 0; - SSyncNode* pSyncNode = taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { taosReleaseRef(tsNodeRefId, rid); @@ -657,8 +655,8 @@ int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak) { return -1; } ASSERT(rid == pSyncNode->rid); - ret = syncNodePropose(pSyncNode, pMsg, isWeak); + int32_t ret = syncNodePropose(pSyncNode, pMsg, isWeak); taosReleaseRef(tsNodeRefId, pSyncNode->rid); return ret; } @@ -669,15 +667,14 @@ int32_t syncProposeBatch(int64_t rid, SRpcMsg* pMsgArr, bool* pIsWeakArr, int32_ return -1; } - int32_t ret = 0; SSyncNode* pSyncNode = taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { terrno = TSDB_CODE_SYN_INTERNAL_ERROR; return -1; } ASSERT(rid == pSyncNode->rid); - ret = syncNodeProposeBatch(pSyncNode, pMsgArr, pIsWeakArr, arrSize); + int32_t ret = syncNodeProposeBatch(pSyncNode, pMsgArr, pIsWeakArr, arrSize); taosReleaseRef(tsNodeRefId, pSyncNode->rid); return ret; } @@ -1529,12 +1526,14 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) { if (pSyncNode != NULL && pSyncNode->pRaftCfg != NULL && pSyncNode->pRaftStore != NULL) { snprintf(logBuf, sizeof(logBuf), "vgId:%d, sync %s %s, term:%lu, commit:%ld, beginlog:%ld, lastlog:%ld, lastsnapshot:%ld, standby:%d, " + "strategy:%d, batch:%d, " "replica-num:%d, " "lconfig:%ld, changing:%d, restore:%d, %s", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex, - pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, - pSyncNode->changing, pSyncNode->restoreFinish, printStr); + pSyncNode->pRaftCfg->isStandBy, pSyncNode->pRaftCfg->snapshotStrategy, pSyncNode->pRaftCfg->batchSize, + pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, pSyncNode->changing, + pSyncNode->restoreFinish, printStr); } else { snprintf(logBuf, sizeof(logBuf), "%s", str); } @@ -1546,12 +1545,14 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) { if (pSyncNode != NULL && pSyncNode->pRaftCfg != NULL && pSyncNode->pRaftStore != NULL) { snprintf(s, len, "vgId:%d, sync %s %s, term:%lu, commit:%ld, beginlog:%ld, lastlog:%ld, lastsnapshot:%ld, standby:%d, " + "strategy:%d, batch:%d, " "replica-num:%d, " "lconfig:%ld, changing:%d, restore:%d, %s", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex, - pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, - pSyncNode->changing, pSyncNode->restoreFinish, printStr); + pSyncNode->pRaftCfg->isStandBy, pSyncNode->pRaftCfg->snapshotStrategy, pSyncNode->pRaftCfg->batchSize, + pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, pSyncNode->changing, + pSyncNode->restoreFinish, printStr); } else { snprintf(s, len, "%s", str); } diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index e53b9d5e36..697c25f304 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -1605,7 +1605,7 @@ void syncAppendEntriesLog2(char* s, const SyncAppendEntries* pMsg) { SyncAppendEntriesBatch* syncAppendEntriesBatchBuild(SSyncRaftEntry** entryPArr, int32_t arrSize, int32_t vgId) { ASSERT(entryPArr != NULL); - ASSERT(arrSize > 0); + ASSERT(arrSize >= 0); int32_t dataLen = 0; int32_t metaArrayLen = sizeof(SOffsetAndContLen) * arrSize; // diff --git a/source/libs/sync/src/syncRaftCfg.c b/source/libs/sync/src/syncRaftCfg.c index ead1168632..6c381f6e7d 100644 --- a/source/libs/sync/src/syncRaftCfg.c +++ b/source/libs/sync/src/syncRaftCfg.c @@ -101,7 +101,7 @@ cJSON *syncCfg2Json(SSyncCfg *pSyncCfg) { char *syncCfg2Str(SSyncCfg *pSyncCfg) { cJSON *pJson = syncCfg2Json(pSyncCfg); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -109,7 +109,7 @@ char *syncCfg2Str(SSyncCfg *pSyncCfg) { char *syncCfg2SimpleStr(SSyncCfg *pSyncCfg) { if (pSyncCfg != NULL) { int32_t len = 512; - char * s = taosMemoryMalloc(len); + char *s = taosMemoryMalloc(len); memset(s, 0, len); snprintf(s, len, "{replica-num:%d, my-index:%d, ", pSyncCfg->replicaNum, pSyncCfg->myIndex); @@ -206,7 +206,7 @@ cJSON *raftCfg2Json(SRaftCfg *pRaftCfg) { char *raftCfg2Str(SRaftCfg *pRaftCfg) { cJSON *pJson = raftCfg2Json(pRaftCfg); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -285,7 +285,7 @@ int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg) { (pRaftCfg->configIndexArr)[i] = atoll(pIndex->valuestring); } - cJSON * pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg"); + cJSON *pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg"); int32_t code = syncCfgFromJson(pJsonSyncCfg, &(pRaftCfg->cfg)); ASSERT(code == 0); diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index da31e9c4c4..17d9aeaea0 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -148,9 +148,17 @@ int32_t syncNodeAppendEntriesPeersSnapshot2(SSyncNode* pSyncNode) { // get entry batch int32_t getCount = 0; SyncIndex getEntryIndex = nextIndex; - for (int32_t i = 0; i < pSyncNode->batchSize; ++i) { + for (int32_t i = 0; i < pSyncNode->pRaftCfg->batchSize; ++i) { SSyncRaftEntry* pEntry = NULL; int32_t code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, getEntryIndex, &pEntry); + + // event log + do { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "get index:%d, code:%d, %s", getEntryIndex, code, tstrerror(terrno)); + syncNodeEventLog(pSyncNode, logBuf); + } while (0); + if (code == 0) { ASSERT(pEntry != NULL); entryPArr[i] = pEntry; @@ -162,12 +170,19 @@ int32_t syncNodeAppendEntriesPeersSnapshot2(SSyncNode* pSyncNode) { } } + // event log + do { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "build batch:%d", getCount); + syncNodeEventLog(pSyncNode, logBuf); + } while (0); + // build msg SyncAppendEntriesBatch* pMsg = syncAppendEntriesBatchBuild(entryPArr, getCount, pSyncNode->vgId); ASSERT(pMsg != NULL); // free entries - for (int32_t i = 0; i < pSyncNode->batchSize; ++i) { + for (int32_t i = 0; i < pSyncNode->pRaftCfg->batchSize; ++i) { SSyncRaftEntry* pEntry = entryPArr[i]; if (pEntry != NULL) { syncEntryDestory(pEntry); diff --git a/tests/script/tsim/sync/vnodesnapshot.sim b/tests/script/tsim/sync/vnodesnapshot.sim index 7ad3afe309..da97f40f1a 100644 --- a/tests/script/tsim/sync/vnodesnapshot.sim +++ b/tests/script/tsim/sync/vnodesnapshot.sim @@ -145,9 +145,9 @@ system sh/exec.sh -n dnode4 -s stop -x SIGINT sql insert into ct1 values(now+0s, 10, 2.0, 3.0) sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) -sql flush database db; +#sql flush database db; -system sh/exec.sh -n dnode4 -s start +#system sh/exec.sh -n dnode4 -s start sql insert into ct1 values(now+1s, 81, 8.1, 8.1)(now+2s, -92, -9.2, -9.2)(now+3s, -73, -7.3, -7.3) From 16c1df91ebd2fa31cbc636b6d578fbac7fc51a37 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 7 Jul 2022 09:07:14 +0000 Subject: [PATCH 15/38] fix: tsdb commit merge bug --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 6 +++--- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 ++ source/dnode/vnode/src/tsdb/tsdbUtil.c | 12 ++++++++---- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index b60a4697fa..8a189d98b3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -602,7 +602,7 @@ static int32_t tsdbGetOvlpNRow(STbDataIter *pIter, SBlock *pBlock) { iter.pRow = NULL; while (true) { - pRow = tsdbTbDataIterGet(pIter); + pRow = tsdbTbDataIterGet(&iter); if (pRow == NULL) break; key = TSDBROW_KEY(pRow); @@ -610,7 +610,7 @@ static int32_t tsdbGetOvlpNRow(STbDataIter *pIter, SBlock *pBlock) { c = tBlockCmprFn(&(SBlock){.maxKey = key, .minKey = key}, pBlock); if (c == 0) { nRow++; - tsdbTbDataIterNext(pIter); + tsdbTbDataIterNext(&iter); } else if (c > 0) { break; } else { @@ -635,7 +635,7 @@ static int32_t tsdbMergeAsSubBlock(SCommitter *pCommitter, STbDataIter *pIter, S code = tsdbCommitterUpdateRowSchema(pCommitter, pBlockIdx->suid, pBlockIdx->uid, TSDBROW_SVERSION(pRow)); if (code) goto _err; while (true) { - if (pRow) break; + if (pRow == NULL) break; code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema); if (code) goto _err; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 2bd62348c9..8e9cd5268e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1106,6 +1106,8 @@ int32_t tsdbReadBlockData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *p SBlockData *pBlockData1 = &(SBlockData){0}; SBlockData *pBlockData2 = &(SBlockData){0}; + tBlockDataInit(pBlockData1); + tBlockDataInit(pBlockData2); for (iSubBlock = 1; iSubBlock < pBlock->nSubBlock; iSubBlock++) { code = tsdbReadSubBlockData(pReader, pBlockIdx, pBlock, iSubBlock, pBlockData1, ppBuf1, ppBuf2); if (code) { diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index f906ef1b54..24b7c08476 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -821,16 +821,20 @@ int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) { int32_t code = 0; int32_t size; + ASSERT(pColDataSrc->nVal > 0); + pColDataDest->cid = pColDataSrc->cid; pColDataDest->type = pColDataSrc->type; pColDataDest->smaOn = pColDataSrc->smaOn; pColDataDest->nVal = pColDataSrc->nVal; pColDataDest->flag = pColDataSrc->flag; - size = BIT2_SIZE(pColDataSrc->nVal); - code = tRealloc(&pColDataDest->pBitMap, size); - if (code) goto _exit; - memcpy(pColDataDest->pBitMap, pColDataSrc->pBitMap, size); + if (pColDataSrc->flag != HAS_NONE && pColDataSrc->flag != HAS_NULL && pColDataSrc->flag != HAS_VALUE) { + size = BIT2_SIZE(pColDataSrc->nVal); + code = tRealloc(&pColDataDest->pBitMap, size); + if (code) goto _exit; + memcpy(pColDataDest->pBitMap, pColDataSrc->pBitMap, size); + } if (IS_VAR_DATA_TYPE(pColDataDest->type)) { size = sizeof(int32_t) * pColDataSrc->nVal; From 211985f03ea41426f7e88d4cc777d212baef2eea Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 7 Jul 2022 17:23:54 +0800 Subject: [PATCH 16/38] refactor: do some internal refactor. --- include/common/tcommon.h | 2 -- source/dnode/vnode/src/tsdb/tsdbRead.c | 4 ++-- source/libs/executor/src/executorimpl.c | 6 +++--- source/libs/function/src/builtinsimpl.c | 20 ++++++++++++++++++-- tests/system-test/failed.txt | 1 + 5 files changed, 24 insertions(+), 9 deletions(-) create mode 100644 tests/system-test/failed.txt diff --git a/include/common/tcommon.h b/include/common/tcommon.h index fd4ed6b180..7eff58d28f 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -88,8 +88,6 @@ typedef struct { #pragma pack(push, 1) typedef struct SColumnDataAgg { int16_t colId; - int16_t minIndex; - int16_t maxIndex; int16_t numOfNull; int64_t sum; int64_t max; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index bd6306e48d..377561baa4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2801,11 +2801,11 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS if (pAgg->colId == pSup->colIds[j]) { if (IS_BSMA_ON(&(pReader->pSchema->columns[i]))) { pSup->plist[j] = pAgg; - i += 1; - j += 1; } else { *allHave = false; } + i += 1; + j += 1; } else if (pAgg->colId < pSup->colIds[j]) { i += 1; } else if (pSup->colIds[j] < pAgg->colId) { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index c3a125da66..f1cca70212 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -741,10 +741,10 @@ static int32_t doCreateConstantValColumnAggInfo(SInputColumnInfoData* pInput, SF if (type == TSDB_DATA_TYPE_BIGINT) { int64_t v = pFuncParam->param.i; - *da = (SColumnDataAgg){.numOfNull = 0, .min = v, .max = v, .maxIndex = 0, .minIndex = 0, .sum = v * numOfRows}; + *da = (SColumnDataAgg){.numOfNull = 0, .min = v, .max = v, .sum = v * numOfRows}; } else if (type == TSDB_DATA_TYPE_DOUBLE) { double v = pFuncParam->param.d; - *da = (SColumnDataAgg){.numOfNull = 0, .maxIndex = 0, .minIndex = 0}; + *da = (SColumnDataAgg){.numOfNull = 0}; *(double*)&da->min = v; *(double*)&da->max = v; @@ -752,7 +752,7 @@ static int32_t doCreateConstantValColumnAggInfo(SInputColumnInfoData* pInput, SF } else if (type == TSDB_DATA_TYPE_BOOL) { // todo validate this data type bool v = pFuncParam->param.i; - *da = (SColumnDataAgg){.numOfNull = 0, .maxIndex = 0, .minIndex = 0}; + *da = (SColumnDataAgg){.numOfNull = 0}; *(bool*)&da->min = 0; *(bool*)&da->max = v; *(bool*)&da->sum = v * numOfRows; diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 03532caac0..d8fcca30c0 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -1080,6 +1080,19 @@ bool getMinmaxFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { static void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos); static void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos); +static int32_t findRowIndex(int32_t start, int32_t num, SColumnInfoData* pCol, const char* tval) { + // the data is loaded, not only the block SMA value + for(int32_t i = start; i < num + start; ++i) { + char* p = colDataGetData(pCol, i); + if (memcpy((void*)tval, p, pCol->info.bytes) == 0) { + return i; + } + } + + ASSERT(0); +} + + int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { int32_t numOfElems = 0; @@ -1111,15 +1124,14 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (isMinFunc) { tval = &pInput->pColumnDataAgg[0]->min; - index = pInput->pColumnDataAgg[0]->minIndex; } else { tval = &pInput->pColumnDataAgg[0]->max; - index = pInput->pColumnDataAgg[0]->maxIndex; } if (!pBuf->assign) { pBuf->v = *(int64_t*)tval; if (pCtx->subsidiaries.num > 0) { + index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); } } else { @@ -1131,6 +1143,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if ((prev < val) ^ isMinFunc) { pBuf->v = val; if (pCtx->subsidiaries.num > 0) { + index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); } } @@ -1143,6 +1156,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if ((prev < val) ^ isMinFunc) { pBuf->v = val; if (pCtx->subsidiaries.num > 0) { + index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); } } @@ -1154,6 +1168,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if ((prev < val) ^ isMinFunc) { pBuf->v = val; if (pCtx->subsidiaries.num > 0) { + index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); } } @@ -1167,6 +1182,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { } if (pCtx->subsidiaries.num > 0) { + index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); } } diff --git a/tests/system-test/failed.txt b/tests/system-test/failed.txt new file mode 100644 index 0000000000..d0b66b1769 --- /dev/null +++ b/tests/system-test/failed.txt @@ -0,0 +1 @@ +#python3 ./test.py -f 2-query/last.py -Q 3 From ce6e505084748dfb43db0a348c53825a20bbf2c7 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 7 Jul 2022 17:27:49 +0800 Subject: [PATCH 17/38] feat(stream): stream state support delete --- source/libs/executor/src/timewindowoperator.c | 16 +++++++++++----- source/util/src/tpagedbuf.c | 4 ++-- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index a89e166e8a..9763f86838 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -3189,7 +3189,7 @@ SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY start *pIndex = index + 1; return pWin; } else if (endTs != INT64_MIN && isInWindow(pWin, endTs, gap)) { - *pIndex = index; + *pIndex = index + 1; return pWin; } } @@ -3442,7 +3442,7 @@ void deleteWindow(SArray* pWinInfos, int32_t index) { taosArrayRemove(pWinInfos, index); } -static void doDeleteSessionWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, int64_t gap, SArray* result) { +static void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, int64_t gap, SArray* result) { SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); TSKEY* startDatas = (TSKEY*)pStartTsCol->pData; SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); @@ -3700,13 +3700,13 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) { SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); // gap must be 0 - doDeleteSessionWindows(&pInfo->streamAggSup, pBlock, 0, pWins); + doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins); if (IS_FINAL_OP(pInfo)) { int32_t childIndex = getChildIndex(pBlock); SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info; // gap must be 0 - doDeleteSessionWindows(&pChildInfo->streamAggSup, pBlock, 0, NULL); + doDeleteTimeWindows(&pChildInfo->streamAggSup, pBlock, 0, NULL); rebuildTimeWindow(pInfo, pWins, pBlock->info.groupId, pOperator->exprSupp.numOfExprs, pOperator); } copyDeleteWindowInfo(pWins, pInfo->pStDeleted); @@ -3840,7 +3840,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { break; } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) { // gap must be 0 - doDeleteSessionWindows(&pInfo->streamAggSup, pBlock, 0, NULL); + doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, NULL); copyDataBlock(pInfo->pDelRes, pBlock); pInfo->pDelRes->info.type = STREAM_DELETE_RESULT; break; @@ -4232,6 +4232,12 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { doClearStateWindows(&pInfo->streamAggSup, pBlock, pInfo->primaryTsIndex, &pInfo->stateCol, pInfo->stateCol.slotId, pSeUpdated, pInfo->pSeDeleted); continue; + } else if (pBlock->info.type == STREAM_DELETE_DATA) { + SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); + doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins); + copyDeleteWindowInfo(pWins, pInfo->pSeDeleted); + taosArrayDestroy(pWins); + continue; } else if (pBlock->info.type == STREAM_GET_ALL) { getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForState); continue; diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index f50fd22ba6..0e608d0da2 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -518,8 +518,8 @@ void releaseBufPageInfo(SDiskbasedBuf* pBuf, SPageInfo* pi) { #ifdef BUF_PAGE_DEBUG uDebug("page_releaseBufPageInfo pageId:%d, used:%d, offset:%"PRId64, pi->pageId, pi->used, pi->offset); #endif - assert(pi->pData != NULL && pi->used == true); -// assert(pi->pData != NULL); + // assert(pi->pData != NULL && pi->used == true); + assert(pi->pData != NULL); pi->used = false; pBuf->statis.releasePages += 1; } From 3ed0280025aeb43a667584b9b0e672cc7e086d0b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 7 Jul 2022 17:39:49 +0800 Subject: [PATCH 18/38] feat: support del index when drop child table --- source/dnode/vnode/src/meta/metaTable.c | 83 +++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index b3cdf47302..341173103c 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -16,6 +16,7 @@ #include "meta.h" static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); +static int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); static int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME); static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME); @@ -103,6 +104,64 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const #endif return 0; } +int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema) { +#ifdef USE_INVERTED_INDEX + if (pMeta->pTagIvtIdx == NULL || pCtbEntry == NULL) { + return -1; + } + void * data = pCtbEntry->ctbEntry.pTags; + const char *tagName = pSchema->name; + + tb_uid_t suid = pCtbEntry->ctbEntry.suid; + tb_uid_t tuid = pCtbEntry->uid; + const void *pTagData = pCtbEntry->ctbEntry.pTags; + int32_t nTagData = 0; + + SArray *pTagVals = NULL; + if (tTagToValArray((const STag *)data, &pTagVals) != 0) { + return -1; + } + + SIndexMultiTerm *terms = indexMultiTermCreate(); + int16_t nCols = taosArrayGetSize(pTagVals); + for (int i = 0; i < nCols; i++) { + STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i); + char type = pTagVal->type; + + char * key = pTagVal->pKey; + int32_t nKey = strlen(key); + + SIndexTerm *term = NULL; + if (type == TSDB_DATA_TYPE_NULL) { + term = indexTermCreate(suid, DEL_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, NULL, 0); + } else if (type == TSDB_DATA_TYPE_NCHAR) { + if (pTagVal->nData > 0) { + char * val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE); + int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE); + memcpy(val, (uint16_t *)&len, VARSTR_HEADER_SIZE); + type = TSDB_DATA_TYPE_VARCHAR; + term = indexTermCreate(suid, DEL_VALUE, type, key, nKey, val, len); + } else if (pTagVal->nData == 0) { + term = indexTermCreate(suid, DEL_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, pTagVal->pData, 0); + } + } else if (type == TSDB_DATA_TYPE_DOUBLE) { + double val = *(double *)(&pTagVal->i64); + int len = sizeof(val); + term = indexTermCreate(suid, DEL_VALUE, type, key, nKey, (const char *)&val, len); + } else if (type == TSDB_DATA_TYPE_BOOL) { + int val = *(int *)(&pTagVal->i64); + int len = sizeof(val); + term = indexTermCreate(suid, DEL_VALUE, TSDB_DATA_TYPE_BOOL, key, nKey, (const char *)&val, len); + } + if (term != NULL) { + indexMultiTermAdd(terms, term); + } + } + indexJsonPut(pMeta->pTagIvtIdx, terms, tuid); + indexMultiTermDestroy(terms); +#endif + return 0; +} int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { SMetaEntry me = {0}; @@ -434,9 +493,33 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { if (type) *type = e.type; + if (e.type == TSDB_CHILD_TABLE) { + void *tData = NULL; + int tLen = 0; + + if (tdbTbGet(pMeta->pUidIdx, &e.ctbEntry.suid, sizeof(tb_uid_t), &tData, &tLen) == 0) { + version = *(int64_t *)tData; + STbDbKey tbDbKey = {.uid = e.ctbEntry.suid, .version = version}; + if (tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &tData, &tLen) == 0) { + SDecoder tdc = {0}; + SMetaEntry stbEntry = {0}; + + tDecoderInit(&tdc, tData, tLen); + metaDecodeEntry(&tdc, &stbEntry); + const SSchema *pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[0]; + if (pTagColumn->type == TSDB_DATA_TYPE_JSON) { + metaDelJsonVarFromIdx(pMeta, &e, pTagColumn); + } + tDecoderClear(&tdc); + } + tdbFree(tData); + } + } + tdbTbDelete(pMeta->pTbDb, &(STbDbKey){.version = version, .uid = uid}, sizeof(STbDbKey), &pMeta->txn); tdbTbDelete(pMeta->pNameIdx, e.name, strlen(e.name) + 1, &pMeta->txn); tdbTbDelete(pMeta->pUidIdx, &uid, sizeof(uid), &pMeta->txn); + if (e.type != TSDB_SUPER_TABLE) metaDeleteTtlIdx(pMeta, &e); if (e.type == TSDB_CHILD_TABLE) { From 74501e8fd5fe2412134bf54cace4c3ffd76a66f3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 7 Jul 2022 09:48:54 +0000 Subject: [PATCH 19/38] fix: merge commit --- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 8e9cd5268e..98decfe002 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -969,6 +969,8 @@ int32_t tsdbReadColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBl SBlockData *pBlockData1 = &(SBlockData){0}; SBlockData *pBlockData2 = &(SBlockData){0}; + tBlockDataInit(pBlockData1); + tBlockDataInit(pBlockData2); for (int32_t iSubBlock = 1; iSubBlock < pBlock->nSubBlock; iSubBlock++) { code = tsdbReadSubColData(pReader, pBlockIdx, pBlock, iSubBlock, aColId, nCol, pBlockData1, ppBuf1, ppBuf2); if (code) goto _err; From 866e4c4be0471ec85798de5c5ab2c893305b4621 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 7 Jul 2022 17:16:12 +0800 Subject: [PATCH 20/38] refactor(stream): internal refactor --- include/common/tcommon.h | 14 ++ include/libs/executor/executor.h | 9 +- include/libs/stream/tstream.h | 5 - source/dnode/vnode/inc/vnode.h | 64 +++++---- source/dnode/vnode/src/inc/tq.h | 23 +--- source/dnode/vnode/src/sma/smaRollup.c | 8 +- source/dnode/vnode/src/tq/tq.c | 24 +++- source/dnode/vnode/src/tq/tqExec.c | 8 +- source/dnode/vnode/src/tq/tqMeta.c | 4 +- source/dnode/vnode/src/tq/tqRead.c | 172 ++++++++++++++++-------- source/libs/executor/inc/executorimpl.h | 7 +- source/libs/executor/src/executor.c | 10 +- source/libs/executor/src/executorMain.c | 31 +++++ source/libs/executor/src/executorimpl.c | 20 +-- source/libs/executor/src/scanoperator.c | 41 +++--- 15 files changed, 276 insertions(+), 164 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index fd4ed6b180..63167bd834 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -117,6 +117,20 @@ typedef struct SSDataBlock { SDataBlockInfo info; } SSDataBlock; +enum { + FETCH_TYPE__DATA = 1, + FETCH_TYPE__META, + FETCH_TYPE__NONE, +}; + +typedef struct { + int8_t fetchType; + union { + SSDataBlock data; + void* meta; + }; +} SFetchRet; + typedef struct SVarColAttr { int32_t* offset; // start position for each entry in the list uint32_t length; // used buffer size that contain the valid data diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 7b4565a99f..0094afef12 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -30,7 +30,7 @@ struct SRpcMsg; struct SSubplan; typedef struct SReadHandle { - void* streamReader; + void* tqReader; void* meta; void* config; void* vnode; @@ -38,7 +38,7 @@ typedef struct SReadHandle { SMsgCb* pMsgCb; bool initMetaReader; bool initTableReader; - bool initStreamReader; + bool initTqReader; } SReadHandle; typedef enum { @@ -52,7 +52,7 @@ typedef enum { * @param streamReadHandle * @return */ -qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle); +qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers); /** * Switch the stream scan to snapshot mode @@ -176,6 +176,9 @@ int32_t qGetStreamScanStatus(qTaskInfo_t tinfo, uint64_t* uid, int64_t* ts); int32_t qStreamPrepareScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts); +void* qExtractReaderFromStreamScanner(void* scanner); +int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner); + #ifdef __cplusplus } #endif diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 52f671e176..d6cb2c27b0 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -235,11 +235,6 @@ typedef struct SStreamTask { int8_t taskStatus; int8_t execStatus; - // exec info - int64_t enqueueVer; - int64_t processedVer; - int64_t checkpointVer; - // node info int32_t selfChildId; int32_t nodeId; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 722087ea45..b381046ecf 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -64,8 +64,8 @@ int32_t vnodeSnapshotRead(SVSnapshotReader *pReader, const void **ppData, uint32 int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen); int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list); int32_t vnodeGetCtbIdList(SVnode *pVnode, int64_t suid, SArray *list); -void * vnodeGetIdx(SVnode *pVnode); -void * vnodeGetIvtIdx(SVnode *pVnode); +void *vnodeGetIdx(SVnode *pVnode); +void *vnodeGetIvtIdx(SVnode *pVnode); int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad); int32_t vnodeValidateTableHash(SVnode *pVnode, char *tableFName); @@ -95,7 +95,7 @@ typedef struct SMetaFltParam { tb_uid_t suid; int16_t cid; int16_t type; - char * val; + char *val; bool reverse; int (*filterFunc)(void *a, void *b, int16_t type); @@ -136,8 +136,8 @@ SArray *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdLis int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond, int32_t tWinIdx); int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); int64_t tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle); -void * tsdbGetIdx(SMeta *pMeta); -void * tsdbGetIvtIdx(SMeta *pMeta); +void *tsdbGetIdx(SMeta *pMeta); +void *tsdbGetIvtIdx(SMeta *pMeta); int32_t tsdbLastRowReaderOpen(void *pVnode, int32_t type, SArray *pTableIdList, int32_t *colId, int32_t numOfCols, void **pReader); @@ -146,19 +146,37 @@ int32_t tsdbLastrowReaderClose(void *pReader); // tq -typedef struct STqReadHandle SStreamReader; +typedef struct STqReader { + int64_t ver; + const SSubmitReq *pMsg; + SSubmitBlk *pBlock; + SSubmitMsgIter msgIter; + SSubmitBlkIter blkIter; -SStreamReader *tqInitSubmitMsgScanner(SMeta *pMeta); + SWalReader *pWalReader; -void tqReadHandleSetColIdList(SStreamReader *pReadHandle, SArray *pColIdList); -int32_t tqReadHandleSetTbUidList(SStreamReader *pHandle, const SArray *tbUidList); -int32_t tqReadHandleAddTbUidList(SStreamReader *pHandle, const SArray *tbUidList); -int32_t tqReadHandleRemoveTbUidList(SStreamReader *pHandle, const SArray *tbUidList); + SMeta *pVnodeMeta; + SHashObj *tbIdHash; + SArray *pColIdList; // SArray -int32_t tqReadHandleSetMsg(SStreamReader *pHandle, SSubmitReq *pMsg, int64_t ver); -bool tqNextDataBlock(SStreamReader *pHandle); -bool tqNextDataBlockFilterOut(SStreamReader *pHandle, SHashObj *filterOutUids); -int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, SStreamReader *pHandle); + int32_t cachedSchemaVer; + int64_t cachedSchemaSuid; + SSchemaWrapper *pSchemaWrapper; + STSchema *pSchema; +} STqReader; + +STqReader *tqOpenReader(SVnode *pVnode); +void tqCloseReader(STqReader *); + +void tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList); +int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList); +int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *tbUidList); +int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); + +int32_t tqReaderSetDataMsg(STqReader *pReader, SSubmitReq *pMsg, int64_t ver); +bool tqNextDataBlock(STqReader *pReader); +bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids); +int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, STqReader *pReader); // sma int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days); @@ -214,7 +232,7 @@ struct SMetaEntry { int8_t type; int8_t flags; // TODO: need refactor? tb_uid_t uid; - char * name; + char *name; union { struct { SSchemaWrapper schemaRow; @@ -225,7 +243,7 @@ struct SMetaEntry { int64_t ctime; int32_t ttlDays; int32_t commentLen; - char * comment; + char *comment; tb_uid_t suid; uint8_t *pTags; } ctbEntry; @@ -233,7 +251,7 @@ struct SMetaEntry { int64_t ctime; int32_t ttlDays; int32_t commentLen; - char * comment; + char *comment; int32_t ncid; // next column id SSchemaWrapper schemaRow; } ntbEntry; @@ -247,17 +265,17 @@ struct SMetaEntry { struct SMetaReader { int32_t flags; - SMeta * pMeta; + SMeta *pMeta; SDecoder coder; SMetaEntry me; - void * pBuf; + void *pBuf; int32_t szBuf; }; struct SMTbCursor { - TBC * pDbc; - void * pKey; - void * pVal; + TBC *pDbc; + void *pKey; + void *pVal; int32_t kLen; int32_t vLen; SMetaReader mr; diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 12fb500ba3..9c3bd85c71 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -44,25 +44,6 @@ extern "C" { typedef struct STqOffsetStore STqOffsetStore; -// tqRead - -struct STqReadHandle { - int64_t ver; - const SSubmitReq* pMsg; - SSubmitBlk* pBlock; - SSubmitMsgIter msgIter; - SSubmitBlkIter blkIter; - - SMeta* pVnodeMeta; - SHashObj* tbIdHash; - SArray* pColIdList; // SArray - - int32_t cachedSchemaVer; - int64_t cachedSchemaSuid; - SSchemaWrapper* pSchemaWrapper; - STSchema* pSchema; -}; - // tqPush typedef struct { @@ -102,7 +83,7 @@ typedef struct { typedef struct { int8_t subType; - SStreamReader* pExecReader[5]; + STqReader* pExecReader[5]; union { STqExecCol execCol; STqExecTb execTb; @@ -118,7 +99,7 @@ typedef struct { int32_t epoch; int8_t fetchMeta; - // reader + // TODO remove SWalReader* pWalReader; // push diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 92168a9b92..125490ecd5 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -325,14 +325,14 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con return TSDB_CODE_FAILED; } - SStreamReader *pReadHandle = tqInitSubmitMsgScanner(pMeta); - if (!pReadHandle) { + STqReader *pReader = tqOpenReader(pVnode); + if (!pReader) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _err; } SReadHandle handle = { - .streamReader = pReadHandle, + .tqReader = pReader, .meta = pMeta, .pMsgCb = pMsgCb, .vnode = pVnode, @@ -364,7 +364,7 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con return TSDB_CODE_SUCCESS; _err: tdFreeRSmaInfo(pRSmaInfo); - taosMemoryFree(pReadHandle); + taosMemoryFree(pReader); return TSDB_CODE_FAILED; } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 2dd80c5d52..e21b0fe9e8 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -447,26 +447,38 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { pHandle->fetchMeta = req.withMeta; pHandle->pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); - for (int32_t i = 0; i < 5; i++) { - pHandle->execHandle.pExecReader[i] = tqInitSubmitMsgScanner(pTq->pVnode->pMeta); - } + /*for (int32_t i = 0; i < 5; i++) {*/ + /*pHandle->execHandle.pExecReader[i] = tqOpenReader(pTq->pVnode);*/ + /*}*/ if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { pHandle->execHandle.execCol.qmsg = req.qmsg; req.qmsg = NULL; for (int32_t i = 0; i < 5; i++) { SReadHandle handle = { - .streamReader = pHandle->execHandle.pExecReader[i], + .tqReader = pHandle->execHandle.pExecReader[i], .meta = pTq->pVnode->pMeta, .vnode = pTq->pVnode, .initTableReader = true, + .initTqReader = true, }; pHandle->execHandle.execCol.task[i] = qCreateStreamExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle); ASSERT(pHandle->execHandle.execCol.task[i]); + void* scanner = NULL; + qExtractStreamScanner(pHandle->execHandle.execCol.task[i], &scanner); + ASSERT(scanner); + pHandle->execHandle.pExecReader[i] = qExtractReaderFromStreamScanner(scanner); + ASSERT(pHandle->execHandle.pExecReader[i]); } } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__DB) { + for (int32_t i = 0; i < 5; i++) { + pHandle->execHandle.pExecReader[i] = tqOpenReader(pTq->pVnode); + } pHandle->execHandle.execDb.pFilterOutTbUid = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { + for (int32_t i = 0; i < 5; i++) { + pHandle->execHandle.pExecReader[i] = tqOpenReader(pTq->pVnode); + } pHandle->execHandle.execTb.suid = req.suid; SArray* tbUidList = taosArrayInit(0, sizeof(int64_t)); vnodeGetCtbIdList(pTq->pVnode, req.suid, tbUidList); @@ -476,7 +488,7 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { tqDebug("vg %d, idx %d, uid: %ld", TD_VID(pTq->pVnode), i, tbUid); } for (int32_t i = 0; i < 5; i++) { - tqReadHandleSetTbUidList(pHandle->execHandle.pExecReader[i], tbUidList); + tqReaderSetTbUidList(pHandle->execHandle.pExecReader[i], tbUidList); } taosArrayDestroy(tbUidList); } @@ -532,7 +544,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) { SReadHandle handle = { .meta = pTq->pVnode->pMeta, .vnode = pTq->pVnode, - .initStreamReader = 1, + .initTqReader = 1, }; pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle); } else { diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 9dd2a0258f..16822e6003 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -135,8 +135,8 @@ int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataR } } else if (pExec->subType == TOPIC_SUB_TYPE__TABLE) { pRsp->withSchema = 1; - SStreamReader* pReader = pExec->pExecReader[workerId]; - tqReadHandleSetMsg(pReader, pReq, 0); + STqReader* pReader = pExec->pExecReader[workerId]; + tqReaderSetDataMsg(pReader, pReq, 0); while (tqNextDataBlock(pReader)) { SSDataBlock block = {0}; if (tqRetrieveDataBlock(&block, pReader) < 0) { @@ -153,8 +153,8 @@ int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataR } } else if (pExec->subType == TOPIC_SUB_TYPE__DB) { pRsp->withSchema = 1; - SStreamReader* pReader = pExec->pExecReader[workerId]; - tqReadHandleSetMsg(pReader, pReq, 0); + STqReader* pReader = pExec->pExecReader[workerId]; + tqReaderSetDataMsg(pReader, pReq, 0); while (tqNextDataBlockFilterOut(pReader, pExec->execDb.pFilterOutTbUid)) { SSDataBlock block = {0}; if (tqRetrieveDataBlock(&block, pReader) < 0) { diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index c7e9c8eed6..67fa4ed166 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -79,12 +79,12 @@ int32_t tqMetaOpen(STQ* pTq) { tDecodeSTqHandle(&decoder, &handle); handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); for (int32_t i = 0; i < 5; i++) { - handle.execHandle.pExecReader[i] = tqInitSubmitMsgScanner(pTq->pVnode->pMeta); + handle.execHandle.pExecReader[i] = tqOpenReader(pTq->pVnode); } if (handle.execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { for (int32_t i = 0; i < 5; i++) { SReadHandle reader = { - .streamReader = handle.execHandle.pExecReader[i], + .tqReader = handle.execHandle.pExecReader[i], .meta = pTq->pVnode->pMeta, .pMsgCb = &pTq->pVnode->msgCb, .vnode = pTq->pVnode, diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index bc992b2211..53ef17a6ba 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -15,6 +15,11 @@ #include "tq.h" +int64_t tqScanLog(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal offset) { + /*if ()*/ + return 0; +} + int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** ppCkHead) { int32_t code = 0; taosThreadMutexLock(&pHandle->pWalReader->mutex); @@ -73,53 +78,107 @@ END: return code; } -SStreamReader* tqInitSubmitMsgScanner(SMeta* pMeta) { - SStreamReader* pReadHandle = taosMemoryMalloc(sizeof(SStreamReader)); - if (pReadHandle == NULL) { +STqReader* tqOpenReader(SVnode* pVnode) { + STqReader* pReader = taosMemoryMalloc(sizeof(STqReader)); + if (pReader == NULL) { return NULL; } - pReadHandle->pVnodeMeta = pMeta; - pReadHandle->pMsg = NULL; - pReadHandle->ver = -1; - pReadHandle->pColIdList = NULL; - pReadHandle->cachedSchemaVer = 0; - pReadHandle->cachedSchemaSuid = 0; - pReadHandle->pSchema = NULL; - pReadHandle->pSchemaWrapper = NULL; - pReadHandle->tbIdHash = NULL; - return pReadHandle; + + // TODO open + /*pReader->pWalReader = walOpenReader(pVnode->pWal, NULL);*/ + + pReader->pVnodeMeta = pVnode->pMeta; + pReader->pMsg = NULL; + pReader->ver = -1; + pReader->pColIdList = NULL; + pReader->cachedSchemaVer = 0; + pReader->cachedSchemaSuid = 0; + pReader->pSchema = NULL; + pReader->pSchemaWrapper = NULL; + pReader->tbIdHash = NULL; + return pReader; } -int32_t tqReadHandleSetMsg(SStreamReader* pReadHandle, SSubmitReq* pMsg, int64_t ver) { - pReadHandle->pMsg = pMsg; +void tqCloseReader(STqReader* pReader) { + // close wal reader + // free cached schema + // free hash + taosMemoryFree(pReader); +} - if (tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter) < 0) return -1; +int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { + bool fromProcessedMsg = pReader->pMsg != NULL; + + while (1) { + if (!fromProcessedMsg) { + if (walNextValidMsg(pReader->pWalReader) < 0) { + ret->fetchType = FETCH_TYPE__NONE; + return -1; + } + void* body = pReader->pWalReader->pHead->head.body; + if (pReader->pWalReader->pHead->head.msgType != TDMT_VND_SUBMIT) { + // TODO do filter + ret->fetchType = FETCH_TYPE__META; + ret->meta = pReader->pWalReader->pHead->head.body; + return 0; + } else { + tqReaderSetDataMsg(pReader, body, pReader->pWalReader->pHead->head.version); + } + } + + while (tqNextDataBlock(pReader)) { + memset(&ret->data, 0, sizeof(SSDataBlock)); + int32_t code = tqRetrieveDataBlock(&ret->data, pReader); + if (code != 0 || ret->data.info.rows == 0) { + if (fromProcessedMsg) { + ret->fetchType = FETCH_TYPE__NONE; + return 0; + } else { + break; + } + } + + ret->fetchType = FETCH_TYPE__DATA; + return 0; + } + + if (fromProcessedMsg) { + ret->fetchType = FETCH_TYPE__NONE; + return 0; + } + } +} + +int32_t tqReaderSetDataMsg(STqReader* pReader, SSubmitReq* pMsg, int64_t ver) { + pReader->pMsg = pMsg; + + if (tInitSubmitMsgIter(pMsg, &pReader->msgIter) < 0) return -1; while (true) { - if (tGetSubmitMsgNext(&pReadHandle->msgIter, &pReadHandle->pBlock) < 0) return -1; - if (pReadHandle->pBlock == NULL) break; + if (tGetSubmitMsgNext(&pReader->msgIter, &pReader->pBlock) < 0) return -1; + if (pReader->pBlock == NULL) break; } - if (tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter) < 0) return -1; - pReadHandle->ver = ver; - memset(&pReadHandle->blkIter, 0, sizeof(SSubmitBlkIter)); + if (tInitSubmitMsgIter(pMsg, &pReader->msgIter) < 0) return -1; + pReader->ver = ver; + memset(&pReader->blkIter, 0, sizeof(SSubmitBlkIter)); return 0; } -bool tqNextDataBlock(SStreamReader* pHandle) { - if (pHandle->pMsg == NULL) return false; +bool tqNextDataBlock(STqReader* pReader) { + if (pReader->pMsg == NULL) return false; while (1) { - if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) { + if (tGetSubmitMsgNext(&pReader->msgIter, &pReader->pBlock) < 0) { return false; } - if (pHandle->pBlock == NULL) { - pHandle->pMsg = NULL; + if (pReader->pBlock == NULL) { + pReader->pMsg = NULL; return false; } - if (pHandle->tbIdHash == NULL) { + if (pReader->tbIdHash == NULL) { return true; } - void* ret = taosHashGet(pHandle->tbIdHash, &pHandle->msgIter.uid, sizeof(int64_t)); + void* ret = taosHashGet(pReader->tbIdHash, &pReader->msgIter.uid, sizeof(int64_t)); /*tqDebug("search uid %ld", pHandle->msgIter.uid);*/ if (ret != NULL) { /*tqDebug("find uid %ld", pHandle->msgIter.uid);*/ @@ -129,7 +188,7 @@ bool tqNextDataBlock(SStreamReader* pHandle) { return false; } -bool tqNextDataBlockFilterOut(SStreamReader* pHandle, SHashObj* filterOutUids) { +bool tqNextDataBlockFilterOut(STqReader* pHandle, SHashObj* filterOutUids) { while (1) { if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) { return false; @@ -145,38 +204,38 @@ bool tqNextDataBlockFilterOut(SStreamReader* pHandle, SHashObj* filterOutUids) { return false; } -int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, SStreamReader* pHandle) { +int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader) { // TODO: cache multiple schema - int32_t sversion = htonl(pHandle->pBlock->sversion); - if (pHandle->cachedSchemaSuid == 0 || pHandle->cachedSchemaVer != sversion || - pHandle->cachedSchemaSuid != pHandle->msgIter.suid) { - if (pHandle->pSchema) taosMemoryFree(pHandle->pSchema); - pHandle->pSchema = metaGetTbTSchema(pHandle->pVnodeMeta, pHandle->msgIter.uid, sversion); - if (pHandle->pSchema == NULL) { + int32_t sversion = htonl(pReader->pBlock->sversion); + if (pReader->cachedSchemaSuid == 0 || pReader->cachedSchemaVer != sversion || + pReader->cachedSchemaSuid != pReader->msgIter.suid) { + if (pReader->pSchema) taosMemoryFree(pReader->pSchema); + pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion); + if (pReader->pSchema == NULL) { tqWarn("cannot found tsschema for table: uid: %ld (suid: %ld), version %d, possibly dropped table", - pHandle->msgIter.uid, pHandle->msgIter.suid, pHandle->cachedSchemaVer); + pReader->msgIter.uid, pReader->msgIter.suid, pReader->cachedSchemaVer); /*ASSERT(0);*/ terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; return -1; } - if (pHandle->pSchemaWrapper) tDeleteSSchemaWrapper(pHandle->pSchemaWrapper); - pHandle->pSchemaWrapper = metaGetTableSchema(pHandle->pVnodeMeta, pHandle->msgIter.uid, sversion, true); - if (pHandle->pSchemaWrapper == NULL) { + if (pReader->pSchemaWrapper) tDeleteSSchemaWrapper(pReader->pSchemaWrapper); + pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, true); + if (pReader->pSchemaWrapper == NULL) { tqWarn("cannot found schema wrapper for table: suid: %ld, version %d, possibly dropped table", - pHandle->msgIter.uid, pHandle->cachedSchemaVer); + pReader->msgIter.uid, pReader->cachedSchemaVer); /*ASSERT(0);*/ terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; return -1; } - pHandle->cachedSchemaVer = sversion; - pHandle->cachedSchemaSuid = pHandle->msgIter.suid; + pReader->cachedSchemaVer = sversion; + pReader->cachedSchemaSuid = pReader->msgIter.suid; } - STSchema* pTschema = pHandle->pSchema; - SSchemaWrapper* pSchemaWrapper = pHandle->pSchemaWrapper; + STSchema* pTschema = pReader->pSchema; + SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper; - int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList); + int32_t colNumNeed = taosArrayGetSize(pReader->pColIdList); if (colNumNeed == 0) { int32_t colMeta = 0; @@ -199,7 +258,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, SStreamReader* pHandle) { while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) { SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta]; col_id_t colIdSchema = pColSchema->colId; - col_id_t colIdNeed = *(col_id_t*)taosArrayGet(pHandle->pColIdList, colNeed); + col_id_t colIdNeed = *(col_id_t*)taosArrayGet(pReader->pColIdList, colNeed); if (colIdSchema < colIdNeed) { colMeta++; } else if (colIdSchema > colIdNeed) { @@ -216,7 +275,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, SStreamReader* pHandle) { } } - if (blockDataEnsureCapacity(pBlock, pHandle->msgIter.numOfRows) < 0) { + if (blockDataEnsureCapacity(pBlock, pReader->msgIter.numOfRows) < 0) { goto FAIL; } @@ -227,13 +286,12 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, SStreamReader* pHandle) { STSRow* row; int32_t curRow = 0; - tInitSubmitBlkIter(&pHandle->msgIter, pHandle->pBlock, &pHandle->blkIter); + tInitSubmitBlkIter(&pReader->msgIter, pReader->pBlock, &pReader->blkIter); - pBlock->info.groupId = 0; - pBlock->info.uid = pHandle->msgIter.uid; - pBlock->info.rows = pHandle->msgIter.numOfRows; + pBlock->info.uid = pReader->msgIter.uid; + pBlock->info.rows = pReader->msgIter.numOfRows; - while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) { + while ((row = tGetSubmitBlkNext(&pReader->blkIter)) != NULL) { tdSTSRowIterReset(&iter, row); // get all wanted col of that block for (int32_t i = 0; i < colActual; i++) { @@ -255,9 +313,9 @@ FAIL: return -1; } -void tqReadHandleSetColIdList(SStreamReader* pReadHandle, SArray* pColIdList) { pReadHandle->pColIdList = pColIdList; } +void tqReaderSetColIdList(STqReader* pReadHandle, SArray* pColIdList) { pReadHandle->pColIdList = pColIdList; } -int tqReadHandleSetTbUidList(SStreamReader* pHandle, const SArray* tbUidList) { +int tqReaderSetTbUidList(STqReader* pHandle, const SArray* tbUidList) { if (pHandle->tbIdHash) { taosHashClear(pHandle->tbIdHash); } @@ -276,7 +334,7 @@ int tqReadHandleSetTbUidList(SStreamReader* pHandle, const SArray* tbUidList) { return 0; } -int tqReadHandleAddTbUidList(SStreamReader* pHandle, const SArray* tbUidList) { +int tqReaderAddTbUidList(STqReader* pHandle, const SArray* tbUidList) { if (pHandle->tbIdHash == NULL) { pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); if (pHandle->tbIdHash == NULL) { @@ -293,7 +351,7 @@ int tqReadHandleAddTbUidList(SStreamReader* pHandle, const SArray* tbUidList) { return 0; } -int tqReadHandleRemoveTbUidList(SStreamReader* pHandle, const SArray* tbUidList) { +int tqReaderRemoveTbUidList(STqReader* pHandle, const SArray* tbUidList) { ASSERT(pHandle->tbIdHash != NULL); for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) { diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 9b32560041..85ddfc4b2b 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -365,7 +365,7 @@ typedef struct SStreamScanInfo { int32_t blockType; // current block type int32_t validBlockIndex; // Is current data has returned? uint64_t numOfExec; // execution times - void* streamReader;// stream block reader handle + STqReader* tqReader; int32_t tsArrayIndex; SArray* tsArray; @@ -383,6 +383,11 @@ typedef struct SStreamScanInfo { SSDataBlock* pPullDataRes; // pull data SSDataBlock SSDataBlock* pDeleteDataRes; // delete data SSDataBlock int32_t deleteDataIndex; + + // status for tmq + //SSchemaWrapper schema; + STqOffset offset; + } SStreamScanInfo; typedef struct SSysTableScanInfo { diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 0155cdd416..a83565cbe0 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -45,7 +45,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu pInfo->blockType = type; if (type == STREAM_INPUT__DATA_SUBMIT) { - if (tqReadHandleSetMsg(pInfo->streamReader, input, 0) < 0) { + if (tqReaderSetDataMsg(pInfo->tqReader, input, 0) < 0) { qError("submit msg messed up when initing stream block, %s" PRIx64, id); return TSDB_CODE_QRY_APP_ERROR; } @@ -105,7 +105,7 @@ int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numO return code; } -qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) { +qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers) { if (msg == NULL) { return NULL; } @@ -120,7 +120,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) { } qTaskInfo_t pTaskInfo = NULL; - code = qCreateExecTask(streamReadHandle, 0, 0, plan, &pTaskInfo, NULL, NULL, OPTR_EXEC_MODEL_STREAM); + code = qCreateExecTask(readers, 0, 0, plan, &pTaskInfo, NULL, NULL, OPTR_EXEC_MODEL_STREAM); if (code != TSDB_CODE_SUCCESS) { // TODO: destroy SSubplan & pTaskInfo terrno = code; @@ -174,11 +174,11 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo SArray* qa = filterQualifiedChildTables(pScanInfo, tableIdList); qDebug(" %d qualified child tables added into stream scanner", (int32_t)taosArrayGetSize(qa)); - code = tqReadHandleAddTbUidList(pScanInfo->streamReader, qa); + code = tqReaderAddTbUidList(pScanInfo->tqReader, qa); taosArrayDestroy(qa); } else { // remove the table id in current list qDebug(" %d remove child tables from the stream scanner", (int32_t)taosArrayGetSize(tableIdList)); - code = tqReadHandleRemoveTbUidList(pScanInfo->streamReader, tableIdList); + code = tqReaderRemoveTbUidList(pScanInfo->tqReader, tableIdList); } return code; diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 7f84b4c174..d5e546f466 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -236,6 +236,37 @@ int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t le return decodeOperator(pTaskInfo->pRoot, pInput, len); } +int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + SOperatorInfo* pOperator = pTaskInfo->pRoot; + + while (1) { + uint8_t type = pOperator->operatorType; + if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { + *scanner = pOperator->info; + return 0; + } else { + ASSERT(pOperator->numOfDownstream == 1); + pOperator = pOperator->pDownstream[0]; + } + } +} + +void* qExtractReaderFromStreamScanner(void* scanner) { + SStreamScanInfo* pInfo = scanner; + return (void*)pInfo->tqReader; +} + +const SSchemaWrapper* qExtractSchemaFromStreamScanner(void* scanner) { + SStreamScanInfo* pInfo = scanner; + return pInfo->tqReader->pSchemaWrapper; +} + +const STqOffset* qExtractStatusFromStreamScanner(void* scanner) { + SStreamScanInfo* pInfo = scanner; + return &pInfo->offset; +} + int32_t qStreamPrepareScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index fa84d79f56..9865fbc5cd 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2842,7 +2842,7 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan } int32_t doPrepareScan(SOperatorInfo* pOperator, uint64_t uid, int64_t ts) { - int32_t type = pOperator->operatorType; + uint8_t type = pOperator->operatorType; pOperator->status = OP_OPENED; @@ -4309,11 +4309,11 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else if (QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN == type) { SLastRowScanPhysiNode* pScanNode = (SLastRowScanPhysiNode*)pPhyNode; -// int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId); -// if (code) { -// pTaskInfo->code = code; -// return NULL; -// } + // int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId); + // if (code) { + // pTaskInfo->code = code; + // return NULL; + // } int32_t code = extractTableSchemaVersion(pHandle, pScanNode->uid, pTaskInfo); if (code != TSDB_CODE_SUCCESS) { @@ -4370,8 +4370,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pOptr = createGroupOperatorInfo(ops[0], pExprInfo, num, pResBlock, pColList, pAggNode->node.pConditions, pScalarExprInfo, numOfScalarExpr, pTaskInfo); } else { - pOptr = - createAggregateOperatorInfo(ops[0], pExprInfo, num, pResBlock, pAggNode->node.pConditions, pScalarExprInfo, numOfScalarExpr, pTaskInfo); + pOptr = createAggregateOperatorInfo(ops[0], pExprInfo, num, pResBlock, pAggNode->node.pConditions, + pScalarExprInfo, numOfScalarExpr, pTaskInfo); } } else if (QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL == type || QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) { SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; @@ -4538,7 +4538,8 @@ SArray* extractColumnInfo(SNodeList* pNodeList) { return pList; } -STsdbReader* doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, STableListInfo* pTableListInfo, const char* idstr) { +STsdbReader* doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, + STableListInfo* pTableListInfo, const char* idstr) { int32_t code = getTableList(pHandle->meta, pHandle->vnode, &pTableScanNode->scan, pTableListInfo); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -4782,7 +4783,6 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead (*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &(*pTaskInfo)->tableqinfoList, pPlan->user); - if (NULL == (*pTaskInfo)->pRoot) { code = (*pTaskInfo)->code; goto _complete; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index aae85bbbc1..68cda52b10 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1232,38 +1232,33 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; blockDataCleanup(pInfo->pRes); - while (tqNextDataBlock(pInfo->streamReader)) { + while (tqNextDataBlock(pInfo->tqReader)) { SSDataBlock block = {0}; // todo refactor - int32_t code = tqRetrieveDataBlock(&block, pInfo->streamReader); + int32_t code = tqRetrieveDataBlock(&block, pInfo->tqReader); - uint64_t groupId = block.info.groupId; - uint64_t uid = block.info.uid; - int32_t numOfRows = block.info.rows; - - if (code != TSDB_CODE_SUCCESS || numOfRows == 0) { + if (code != TSDB_CODE_SUCCESS || block.info.rows == 0) { pTaskInfo->code = code; return NULL; } - pInfo->pRes->info.groupId = groupId; - pInfo->pRes->info.rows = numOfRows; - pInfo->pRes->info.uid = uid; + pInfo->pRes->info.rows = block.info.rows; + pInfo->pRes->info.uid = block.info.uid; pInfo->pRes->info.type = STREAM_NORMAL; - pInfo->pRes->info.capacity = numOfRows; + pInfo->pRes->info.capacity = block.info.rows; // for generating rollup SMA result, each time is an independent time serie. // TODO temporarily used, when the statement of "partition by tbname" is ready, remove this if (pInfo->assignBlockUid) { - pInfo->pRes->info.groupId = uid; - } else { - pInfo->pRes->info.groupId = groupId; + pInfo->pRes->info.groupId = block.info.uid; } - uint64_t* groupIdPre = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &uid, sizeof(int64_t)); + uint64_t* groupIdPre = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &block.info.uid, sizeof(int64_t)); if (groupIdPre) { pInfo->pRes->info.groupId = *groupIdPre; + } else { + pInfo->pRes->info.groupId = 0; } // todo extract method @@ -1413,13 +1408,13 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys } } - if (pHandle->initStreamReader) { - ASSERT(pHandle->streamReader == NULL); - pInfo->streamReader = tqInitSubmitMsgScanner(pHandle->meta); - ASSERT(pInfo->streamReader); + if (pHandle->initTqReader) { + ASSERT(pHandle->tqReader == NULL); + pInfo->tqReader = tqOpenReader(pHandle->vnode); + ASSERT(pInfo->tqReader); } else { - ASSERT(pHandle->streamReader); - pInfo->streamReader = pHandle->streamReader; + ASSERT(pHandle->tqReader); + pInfo->tqReader = pHandle->tqReader; } if (pSTInfo->interval.interval > 0) { @@ -1435,9 +1430,9 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->tableUid = pScanPhyNode->uid; // set the extract column id to streamHandle - tqReadHandleSetColIdList(pInfo->streamReader, pColIds); + tqReaderSetColIdList(pInfo->tqReader, pColIds); SArray* tableIdList = extractTableIdList(&pTaskInfo->tableqinfoList); - int32_t code = tqReadHandleSetTbUidList(pInfo->streamReader, tableIdList); + int32_t code = tqReaderSetTbUidList(pInfo->tqReader, tableIdList); if (code != 0) { taosArrayDestroy(tableIdList); goto _error; From e0532065f089e4820d7a0969a3b82af5ce0444ba Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 7 Jul 2022 17:58:03 +0800 Subject: [PATCH 21/38] other: comment out the test case and add debug logs --- source/dnode/vnode/src/sma/smaRollup.c | 2 ++ tests/script/jenkins/basic.txt | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 92168a9b92..669fbe1b49 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -277,6 +277,8 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo pItem->maxDelay = TSDB_MAX_ROLLUP_MAX_DELAY; } pItem->level = (idx == 0 ? TSDB_RETENTION_L1 : TSDB_RETENTION_L2); + smaInfo("vgId:%d table:%" PRIi64 " level:%" PRIi8 " maxdelay:%" PRIi64 " watermark:%" PRIi64 ", finally maxdelay:%"PRIi32, SMA_VID(pSma), + pRSmaInfo->suid, idx + 1, param->maxdelay[idx], param->watermark[idx], pItem->maxDelay); } return TSDB_CODE_SUCCESS; _err: diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 4e009e702d..0783aa0fd1 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -163,8 +163,8 @@ # --- sma ./test.sh -f tsim/sma/drop_sma.sim ./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim -#./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim -#./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim +./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim +./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim # --- valgrind ./test.sh -f tsim/valgrind/checkError1.sim From 9354c8d52045b4f452e2e7d1fed17974d0c944de Mon Sep 17 00:00:00 2001 From: huolibo Date: Thu, 7 Jul 2022 19:13:59 +0800 Subject: [PATCH 22/38] fix: change fetchRawBlock method for tmq --- source/client/src/TMQConnector.c | 6 ++---- source/client/src/TSDBJNIConnector.c | 3 +-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/source/client/src/TMQConnector.c b/source/client/src/TMQConnector.c index c3d1e60782..1d84dcf7a2 100644 --- a/source/client/src/TMQConnector.c +++ b/source/client/src/TMQConnector.c @@ -324,9 +324,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_fetchRawBlockImp( (*env)->CallVoidMethod(env, rowobj, g_blockdataSetNumOfRowsFp, (jint)numOfRows); (*env)->CallVoidMethod(env, rowobj, g_blockdataSetNumOfColsFp, (jint)numOfFields); - char *chars = (char *)data; - int32_t len = chars[0] + (chars[1] << 8) + (chars[2] << 16) + (chars[3] << 24); - (*env)->CallVoidMethod(env, rowobj, g_blockdataSetByteArrayFp, len, jniFromNCharToByteArray(env, (char *)data, len)); - + int32_t len = *(int32_t *)data; + (*env)->CallVoidMethod(env, rowobj, g_blockdataSetByteArrayFp, jniFromNCharToByteArray(env, (char *)data, len)); return JNI_SUCCESS; } diff --git a/source/client/src/TSDBJNIConnector.c b/source/client/src/TSDBJNIConnector.c index 246785b570..227c2fff18 100644 --- a/source/client/src/TSDBJNIConnector.c +++ b/source/client/src/TSDBJNIConnector.c @@ -592,8 +592,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchBlockImp(JNI (*env)->CallVoidMethod(env, rowobj, g_blockdataSetNumOfRowsFp, (jint)numOfRows); (*env)->CallVoidMethod(env, rowobj, g_blockdataSetNumOfColsFp, (jint)numOfFields); - char *chars = (char *)data; - int32_t len = chars[0] + (chars[1] << 8) + (chars[2] << 16) + (chars[3] << 24); + int32_t len = *(int32_t *)data; (*env)->CallVoidMethod(env, rowobj, g_blockdataSetByteArrayFp, jniFromNCharToByteArray(env, (char *)data, len)); return JNI_SUCCESS; From 5c2e98544d3cb15d724ba4031e92af8ee392f642 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 7 Jul 2022 22:40:20 +0800 Subject: [PATCH 23/38] fix(query): check if the data block overlaps with the delete skyline --- source/dnode/vnode/src/tsdb/tsdbRead.c | 43 +++++++++++++++++++++----- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 377561baa4..43315073aa 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1416,29 +1416,58 @@ static bool keyOverlapFileBlock(TSDBKEY key, SBlock* pBlock, SVersionRange* pVer (pBlock->minVersion <= pVerRange->maxVer); } +static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBlock* pBlock) { + if (pBlockScanInfo->delSkyline == NULL) { + return false; + } + + TSDBKEY* pFirst = taosArrayGet(pBlockScanInfo->delSkyline, 0); + TSDBKEY* pLast = taosArrayGetLast(pBlockScanInfo->delSkyline); + + // ts is not overlap + if (pBlock->minKey.ts > pLast->ts || pBlock->maxKey.ts < pFirst->ts) { + return false; + } + + // version is not overlap + size_t num = taosArrayGetSize(pBlockScanInfo->delSkyline); + for(int32_t i = pBlockScanInfo->fileDelIndex; i < num; ++i) { + TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, i); + if (p->ts >= pBlock->minKey.ts && p->ts <= pBlock->maxKey.ts) { + if (p->version >= pBlock->minVersion) { + return true; + } + } else if (p->ts > pBlock->maxKey.ts) { + return false; + } + } + + ASSERT(0); + return false; +} + // 1. the version of all rows should be less than the endVersion // 2. current block should not overlap with next neighbor block // 3. current timestamp should not be overlap with each other // 4. output buffer should be large enough to hold all rows in current block +// 5. delete info should not overlap with current block data static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pFBlock, SBlock* pBlock, STableBlockScanInfo* pScanInfo, TSDBKEY key) { int32_t neighborIndex = 0; SBlock* pNeighbor = getNeighborBlockOfSameTable(pFBlock, pScanInfo, &neighborIndex, pReader->order); + // overlap with neighbor bool overlapWithNeighbor = false; if (pNeighbor) { overlapWithNeighbor = overlapWithNeighborBlock(pBlock, pNeighbor, pReader->order); } - bool hasDup = false; - if (pBlock->nSubBlock == 1) { - hasDup = pBlock->hasDup; - } else { - hasDup = true; - } + // has duplicated ts of different version in this block + bool hasDup = (pBlock->nSubBlock == 1)? pBlock->hasDup:true; + bool overlapWithDel= overlapWithDelSkyline(pScanInfo, pBlock); return (overlapWithNeighbor || hasDup || dataBlockPartiallyRequired(&pReader->window, &pReader->verRange, pBlock) || - keyOverlapFileBlock(key, pBlock, &pReader->verRange) || (pBlock->nRow > pReader->capacity)); + keyOverlapFileBlock(key, pBlock, &pReader->verRange) || (pBlock->nRow > pReader->capacity) || overlapWithDel); } static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, int64_t endKey) { From fe87085b213f1f2b1ae0d3d2753f78f45d724d1f Mon Sep 17 00:00:00 2001 From: Steven Li Date: Thu, 7 Jul 2022 23:09:27 +0000 Subject: [PATCH 24/38] Minor adjustments to crash_gen tool, so that it can work with 3.0 code base, now can do simple -t 4 -s 25 --- tests/pytest/crash_gen.sh | 7 ++++++- tests/pytest/crash_gen/crash_gen_main.py | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/tests/pytest/crash_gen.sh b/tests/pytest/crash_gen.sh index 127e13c5be..539314dea4 100755 --- a/tests/pytest/crash_gen.sh +++ b/tests/pytest/crash_gen.sh @@ -48,7 +48,12 @@ fi PYTHON_EXEC=python3.8 # First we need to set up a path for Python to find our own TAOS modules, so that "import" can work. -export PYTHONPATH=$(pwd)/../../src/connector/python:$(pwd) +# export PYTHONPATH=$(pwd)/../../src/connector/python:$(pwd) +# NOTE: we are now pointing outside the current github, per Wade on 7/7/2022, we'll be keeping connectors outside +# and there does not seem to be a module to reference that. +PROJECT_PARENT=$(pwd)/../../.. +TAOS_PYTHON_PROJECT_DIR=$PROJECT_PARENT/taos-connector-python +export PYTHONPATH=$TAOS_PYTHON_PROJECT_DIR:$(pwd) # Then let us set up the library path so that our compiled SO file can be loaded by Python export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$LIB_DIR diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index b743eee2ef..08155f656b 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -466,6 +466,7 @@ class ThreadCoordinator: self._te = None # No more executor, time to end Logging.debug("Main thread tapping all threads one last time...") self.tapAllThreads() # Let the threads run one last time + #TODO: looks like we are not capturing the failures for the last step yet (i.e. calling registerFailure if neccessary) Logging.debug("\r\n\n--> Main thread ready to finish up...") Logging.debug("Main thread joining all threads") @@ -1290,6 +1291,7 @@ class Task(): def _isErrAcceptable(self, errno, msg): if errno in [ + # TDengine 2.x Error Codes: 0x05, # TSDB_CODE_RPC_NOT_READY 0x0B, # Unable to establish connection, more details in TD-1648 # 0x200, # invalid SQL， TODO: re-examine with TD-934 @@ -1310,6 +1312,18 @@ class Task(): 0x14, # db not ready, errno changed 0x600, # Invalid table ID, why? 0x218, # Table does not exist + + # TDengine 3.0 Error Codes: + 0x0333, # Object is creating # TODO: this really is NOT an acceptable error + 0x03A0, # STable already exists + 0x03A1, # STable [does] not exist + 0x03AA, # Tag already exists + 0x0603, # Table already exists + 0x2602, # Table does not exist + 0x260d, # Tags number not matched + + + 1000 # REST catch-all error ]: return True # These are the ALWAYS-ACCEPTABLE ones @@ -1749,6 +1763,8 @@ class TdSuperTable: tagType = tags[tagName] if tagType == 'BINARY': tagStrs.append("'Beijing-Shanghai-LosAngeles'") + elif tagType== 'VARCHAR': + tagStrs.append("'London-Paris-Berlin'") elif tagType == 'FLOAT': tagStrs.append('9.9') elif tagType == 'INT': From c591d16e29e1345f077b225c78b8e08167aca47b Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Fri, 8 Jul 2022 09:24:53 +0800 Subject: [PATCH 25/38] test: add loop script for multi run use case --- tests/script/loop.sh | 8 +++++- tests/system-test/loop.sh | 55 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) create mode 100755 tests/system-test/loop.sh diff --git a/tests/script/loop.sh b/tests/script/loop.sh index b435eef123..b3bc9728de 100755 --- a/tests/script/loop.sh +++ b/tests/script/loop.sh @@ -13,7 +13,7 @@ CMD_NAME= LOOP_TIMES=5 SLEEP_TIME=0 -while getopts "f:t:s:" arg +while getopts "hf:t:s:" arg do case $arg in f) @@ -25,6 +25,12 @@ do s) SLEEP_TIME=$OPTARG ;; + h) + echo "Usage: $(basename $0) -f [cmd name] " + echo " -t [loop times] " + echo " -s [sleep time] " + exit 0 + ;; ?) echo "unknow argument" ;; diff --git a/tests/system-test/loop.sh b/tests/system-test/loop.sh new file mode 100755 index 0000000000..b3bc9728de --- /dev/null +++ b/tests/system-test/loop.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +################################################## +# +# Do simulation test +# +################################################## + +set -e +#set -x + +CMD_NAME= +LOOP_TIMES=5 +SLEEP_TIME=0 + +while getopts "hf:t:s:" arg +do + case $arg in + f) + CMD_NAME=$OPTARG + ;; + t) + LOOP_TIMES=$OPTARG + ;; + s) + SLEEP_TIME=$OPTARG + ;; + h) + echo "Usage: $(basename $0) -f [cmd name] " + echo " -t [loop times] " + echo " -s [sleep time] " + exit 0 + ;; + ?) + echo "unknow argument" + ;; + esac +done + +echo LOOP_TIMES ${LOOP_TIMES} +echo CMD_NAME ${CMD_NAME} +echo SLEEP_TIME ${SLEEP_TIME} + +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +for ((i=0; i<$LOOP_TIMES; i++ )) +do + echo -e $GREEN loop $i $NC + echo -e $GREEN cmd $CMD_NAME $NC + $CMD_NAME + sleep ${SLEEP_TIME} +done From 2339f9a5b0c79d3d345fc73e24ba09872153569c Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Fri, 8 Jul 2022 10:39:38 +0800 Subject: [PATCH 26/38] test: use flush data into disk,not restart taosd --- .../system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py | 5 +++-- tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py | 5 +++-- .../7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py | 6 +++--- .../system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py | 7 ++++--- tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py | 7 ++++--- tests/system-test/7-tmq/tmqConsFromTsdb.py | 5 +++-- .../system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py | 5 +++-- tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py | 5 +++-- .../7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py | 5 +++-- .../system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py | 5 +++-- tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py | 5 +++-- tests/system-test/7-tmq/tmqConsFromTsdb1.py | 5 +++-- 12 files changed, 38 insertions(+), 27 deletions(-) diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py index bd8531d7c8..d22d183e86 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py @@ -64,8 +64,9 @@ class TDTestCase: startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) tdLog.info("restart taosd to ensure that the data falls into the disk") - tdDnodes.stop(1) - tdDnodes.start(1) + # tdDnodes.stop(1) + # tdDnodes.start(1) + tdSql.query("flush database %s"%(paraDict['dbName'])) return def tmqCase1(self): diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py index 1d2aaccda5..951a747069 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py @@ -64,8 +64,9 @@ class TDTestCase: startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) tdLog.info("restart taosd to ensure that the data falls into the disk") - tdDnodes.stop(1) - tdDnodes.start(1) + # tdDnodes.stop(1) + # tdDnodes.start(1) + tdSql.query("flush database %s"%(paraDict['dbName'])) return def tmqCase1(self): diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py index 2720b7cdce..6ee089af4e 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py @@ -64,9 +64,9 @@ class TDTestCase: startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) tdLog.info("restart taosd to ensure that the data falls into the disk") - tdDnodes.stop(1) - # tdDnodes.start(1) - tdDnodes.starttaosd(1) + # tdDnodes.stop(1) + # tdDnodes.starttaosd(1) + tdSql.query("flush database %s"%(paraDict['dbName'])) return def tmqCase1(self): diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py index 0b2dddd24a..882989bfb6 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py @@ -64,9 +64,10 @@ class TDTestCase: startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) tdLog.info("restart taosd to ensure that the data falls into the disk") - tdDnodes.stop(1) - # tdDnodes.start(1) - tdDnodes.starttaosd(1) + # tdDnodes.stop(1) + # # tdDnodes.start(1) + # tdDnodes.starttaosd(1) + tdSql.query("flush database %s"%(paraDict['dbName'])) return def tmqCase1(self): diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py index a4d6648276..c334ff752b 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py @@ -64,9 +64,10 @@ class TDTestCase: startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) tdLog.info("restart taosd to ensure that the data falls into the disk") - tdDnodes.stop(1) - # tdDnodes.start(1) - tdDnodes.starttaosd(1) + # tdDnodes.stop(1) + # # tdDnodes.start(1) + # tdDnodes.starttaosd(1) + tdSql.query("flush database %s"%(paraDict['dbName'])) return def tmqCase1(self): diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb.py b/tests/system-test/7-tmq/tmqConsFromTsdb.py index c18474dcc3..a4a242365a 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb.py @@ -64,8 +64,9 @@ class TDTestCase: startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) tdLog.info("restart taosd to ensure that the data falls into the disk") - tdDnodes.stop(1) - tdDnodes.start(1) + # tdDnodes.stop(1) + # tdDnodes.start(1) + tdSql.query("flush database %s"%(paraDict['dbName'])) return def tmqCase1(self): diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py index 540c9dbbe1..ec11b3286c 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py @@ -64,8 +64,9 @@ class TDTestCase: startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) tdLog.info("restart taosd to ensure that the data falls into the disk") - tdDnodes.stop(1) - tdDnodes.start(1) + # tdDnodes.stop(1) + # tdDnodes.start(1) + tdSql.query("flush database %s"%(paraDict['dbName'])) return def tmqCase3(self): diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py index 5cb373092b..4878652593 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py @@ -64,8 +64,9 @@ class TDTestCase: startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) tdLog.info("restart taosd to ensure that the data falls into the disk") - tdDnodes.stop(1) - tdDnodes.start(1) + # tdDnodes.stop(1) + # tdDnodes.start(1) + tdSql.query("flush database %s"%(paraDict['dbName'])) return def tmqCase3(self): diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py index eaef134845..451dc43343 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py @@ -64,8 +64,9 @@ class TDTestCase: startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) tdLog.info("restart taosd to ensure that the data falls into the disk") - tdDnodes.stop(1) - tdDnodes.start(1) + # tdDnodes.stop(1) + # tdDnodes.start(1) + tdSql.query("flush database %s"%(paraDict['dbName'])) return def tmqCase3(self): diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py index c1b327e5f1..3b3467f9f9 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py @@ -64,8 +64,9 @@ class TDTestCase: startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) tdLog.info("restart taosd to ensure that the data falls into the disk") - tdDnodes.stop(1) - tdDnodes.start(1) + # tdDnodes.stop(1) + # tdDnodes.start(1) + tdSql.query("flush database %s"%(paraDict['dbName'])) return def tmqCase3(self): diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py index dd8a0ad33a..d1fe69f0b7 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py @@ -64,8 +64,9 @@ class TDTestCase: startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) tdLog.info("restart taosd to ensure that the data falls into the disk") - tdDnodes.stop(1) - tdDnodes.start(1) + # tdDnodes.stop(1) + # tdDnodes.start(1) + tdSql.query("flush database %s"%(paraDict['dbName'])) return def tmqCase3(self): diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1.py b/tests/system-test/7-tmq/tmqConsFromTsdb1.py index a183eda1c7..597f7968ae 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1.py @@ -64,8 +64,9 @@ class TDTestCase: startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) tdLog.info("restart taosd to ensure that the data falls into the disk") - tdDnodes.stop(1) - tdDnodes.start(1) + # tdDnodes.stop(1) + # tdDnodes.start(1) + tdSql.query("flush database %s"%(paraDict['dbName'])) return def tmqCase3(self): From c3398006117274e509bd3ddf70c2cad19196f920 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 8 Jul 2022 11:02:12 +0800 Subject: [PATCH 27/38] fix(query): handle delete duration generating data block. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 115 +++++++++++++++++++------ 1 file changed, 90 insertions(+), 25 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 43315073aa..53579474df 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -137,7 +137,7 @@ static int32_t doMergeRowsInBuf(SIterInfo* pIter, int64_t ts, SArray* pDelList, static int32_t doAppendOneRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow); static void setComposedBlockFlag(STsdbReader* pReader, bool composed); static void updateSchema(TSDBROW* pRow, uint64_t uid, STsdbReader* pReader); -static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey); +static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order); static void doMergeMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, STSRow** pTSRow, STsdbReader* pReader); @@ -1416,7 +1416,7 @@ static bool keyOverlapFileBlock(TSDBKEY key, SBlock* pBlock, SVersionRange* pVer (pBlock->minVersion <= pVerRange->maxVer); } -static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBlock* pBlock) { +static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBlock* pBlock, int32_t order) { if (pBlockScanInfo->delSkyline == NULL) { return false; } @@ -1429,9 +1429,11 @@ static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBl return false; } + int32_t step = ASCENDING_TRAVERSE(order)? 1:-1; + // version is not overlap size_t num = taosArrayGetSize(pBlockScanInfo->delSkyline); - for(int32_t i = pBlockScanInfo->fileDelIndex; i < num; ++i) { + for(int32_t i = pBlockScanInfo->fileDelIndex; i < num; i += step) { TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, i); if (p->ts >= pBlock->minKey.ts && p->ts <= pBlock->maxKey.ts) { if (p->version >= pBlock->minVersion) { @@ -1464,7 +1466,7 @@ static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pFBloc // has duplicated ts of different version in this block bool hasDup = (pBlock->nSubBlock == 1)? pBlock->hasDup:true; - bool overlapWithDel= overlapWithDelSkyline(pScanInfo, pBlock); + bool overlapWithDel= overlapWithDelSkyline(pScanInfo, pBlock, pReader->order); return (overlapWithNeighbor || hasDup || dataBlockPartiallyRequired(&pReader->window, &pReader->verRange, pBlock) || keyOverlapFileBlock(key, pBlock, &pReader->verRange) || (pBlock->nRow > pReader->capacity) || overlapWithDel); @@ -1691,7 +1693,7 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDum } TSDBKEY k = {.ts = ts, .version = ver}; - if (hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->fileDelIndex, &k)) { + if (hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->fileDelIndex, &k, pReader->order)) { return false; } @@ -2220,41 +2222,104 @@ static SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond // taosArrayPush(pTsdbReadHandle->pTableCheckInfo, &info); // } -bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey) { +bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order) { ASSERT(pKey != NULL); if (pDelList == NULL) { return false; } + size_t num = taosArrayGetSize(pDelList); + bool asc = ASCENDING_TRAVERSE(order); + int32_t step = asc? 1:-1; - if (*index >= taosArrayGetSize(pDelList) - 1) { - TSDBKEY* last = taosArrayGetLast(pDelList); - if (pKey->ts > last->ts) { - return false; - } else if (pKey->ts == last->ts) { - size_t size = taosArrayGetSize(pDelList); - TSDBKEY* prev = taosArrayGet(pDelList, size - 2); - if (prev->version >= pKey->version) { - return true; - } else { + if (asc) { + if (*index >= num - 1) { + TSDBKEY* last = taosArrayGetLast(pDelList); + ASSERT(pKey->ts >= last->ts); + + if (pKey->ts > last->ts) { return false; + } else if (pKey->ts == last->ts) { + TSDBKEY* prev = taosArrayGet(pDelList, num - 2); + return (prev->version >= pKey->version); } } else { - ASSERT(0); + TSDBKEY* pCurrent = taosArrayGet(pDelList, *index); + TSDBKEY* pNext = taosArrayGet(pDelList, (*index) + 1); + + if (pKey->ts < pCurrent->ts) { + return false; + } + + if (pCurrent->ts <= pKey->ts && pNext->ts >= pKey->ts && pCurrent->version >= pKey->version) { + return true; + } + + while (pNext->ts <= pKey->ts && (*index) < num - 1) { + (*index) += 1; + + if ((*index) < num - 1) { + pCurrent = taosArrayGet(pDelList, *index); + pNext = taosArrayGet(pDelList, (*index) + 1); + + // it is not a consecutive deletion range, ignore it + if (pCurrent->version == 0 && pNext->version > 0) { + continue; + } + + if (pCurrent->ts <= pKey->ts && pNext->ts >= pKey->ts && pCurrent->version >= pKey->version) { + return true; + } + } + } + + return false; } } else { - TSDBKEY* pCurrent = taosArrayGet(pDelList, *index); - TSDBKEY* pNext = taosArrayGet(pDelList, (*index) + 1); + if (*index <= 0) { + TSDBKEY* pFirst = taosArrayGet(pDelList, 0); - if (pCurrent->ts <= pKey->ts && pNext->ts >= pKey->ts && pCurrent->version >= pKey->version) { - return true; + if (pKey->ts < pFirst->ts) { + return false; + } else if (pKey->ts == pFirst->ts) { + return pFirst->version >= pKey->version; + } else { + ASSERT(0); + } } else { - while (pNext->ts < pKey->ts && (*index) < taosArrayGetSize(pDelList) - 1) { - (*index) += 1; + TSDBKEY* pCurrent = taosArrayGet(pDelList, *index); + TSDBKEY* pPrev = taosArrayGet(pDelList, (*index) - 1); + + if (pKey->ts > pCurrent->ts) { + return false; + } + + if (pPrev->ts <= pKey->ts && pCurrent->ts >= pKey->ts && pPrev->version >= pKey->version) { + return true; + } + + while (pPrev->ts >= pKey->ts && (*index) > 1) { + (*index) += step; + + if ((*index) >= 1) { + pCurrent = taosArrayGet(pDelList, *index); + pPrev = taosArrayGet(pDelList, (*index) - 1); + + // it is not a consecutive deletion range, ignore it + if (pCurrent->version > 0 && pPrev->version == 0) { + continue; + } + + if (pPrev->ts <= pKey->ts && pCurrent->ts >= pKey->ts && pPrev->version >= pKey->version) { + return true; + } + } } return false; } } + + return false; } TSDBROW* getValidRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader) { @@ -2271,7 +2336,7 @@ TSDBROW* getValidRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pRea // it is a valid data version if ((key.version <= pReader->verRange.maxVer && key.version >= pReader->verRange.minVer) && - (!hasBeenDropped(pDelList, &pIter->index, &key))) { + (!hasBeenDropped(pDelList, &pIter->index, &key, pReader->order))) { return pRow; } @@ -2290,7 +2355,7 @@ TSDBROW* getValidRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pRea } if (key.version <= pReader->verRange.maxVer && key.version >= pReader->verRange.minVer && - (!hasBeenDropped(pDelList, &pIter->index, &key))) { + (!hasBeenDropped(pDelList, &pIter->index, &key, pReader->order))) { return pRow; } } From ae19c8f63e2674e36afdc64e293b3013fe0c55b4 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Fri, 8 Jul 2022 11:49:32 +0800 Subject: [PATCH 28/38] test: comment 2 rsma cases --- tests/script/jenkins/basic.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 0783aa0fd1..4e009e702d 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -163,8 +163,8 @@ # --- sma ./test.sh -f tsim/sma/drop_sma.sim ./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim -./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim -./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim +#./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim +#./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim # --- valgrind ./test.sh -f tsim/valgrind/checkError1.sim From 74eaa512d367a1affdb848a34b5c5a050c38b5d8 Mon Sep 17 00:00:00 2001 From: Zhengmao Zhu <70138133+fenghuazzm@users.noreply.github.com> Date: Fri, 8 Jul 2022 12:04:33 +0800 Subject: [PATCH 29/38] docs: Update 03-immigrate.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 存储资源估算的计算公式有误 --- docs/zh/25-application/03-immigrate.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/25-application/03-immigrate.md b/docs/zh/25-application/03-immigrate.md index 9d8946bc4a..d1c9caea09 100644 --- a/docs/zh/25-application/03-immigrate.md +++ b/docs/zh/25-application/03-immigrate.md @@ -367,10 +367,10 @@ WHERE ts>=1510560000 AND ts<=1515000009 ### 存储资源估算 -假设产生数据并需要存储的传感器设备数量为 `n`，数据生成的频率为`t`条/秒，每条记录的长度为 `L` bytes，则每天产生的数据规模为 `n×t×L` bytes。假设压缩比为 C，则每日产生数据规模为 `(n×t×L)/C` bytes。存储资源预估为能够容纳 1.5 年的数据规模，生产环境下 TDengine 的压缩比 C 一般在 5 ~ 7 之间，同时为最后结果增加 20% 的冗余，可计算得到需要存储资源： +假设产生数据并需要存储的传感器设备数量为 `n`，数据生成的频率为`t`条/秒，每条记录的长度为 `L` bytes，则每天产生的数据规模为 `86400×n×t×L` bytes。假设压缩比为 C，则每日产生数据规模为 `(86400×n×t×L)/C` bytes。存储资源预估为能够容纳 1.5 年的数据规模，生产环境下 TDengine 的压缩比 C 一般在 5 ~ 7 之间，同时为最后结果增加 20% 的冗余，可计算得到需要存储资源： ```matlab -(n×t×L)×(365×1.5)×(1+20%)/C +(86400×n×t×L)×(365×1.5)×(1+20%)/C ``` 结合以上的计算公式，将参数带入计算公式，在不考虑标签信息的情况下，每年产生的原始数据规模是 11.8TB。需要注意的是，由于标签信息在 TDengine 中关联到每个时间线，并不是每条记录。所以需要记录的数据量规模相对于产生的数据有一定的降低，而这部分标签数据整体上可以忽略不记。假设压缩比为 5，则保留的数据规模最终为 2.56 TB。 From 81205dedb1fed578fa54818847e38e690c89441a Mon Sep 17 00:00:00 2001 From: Zhengmao Zhu <70138133+fenghuazzm@users.noreply.github.com> Date: Fri, 8 Jul 2022 12:07:21 +0800 Subject: [PATCH 30/38] Update 03-immigrate.md --- docs/en/25-application/03-immigrate.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/25-application/03-immigrate.md b/docs/en/25-application/03-immigrate.md index 4d47aec1d7..fe67f97389 100644 --- a/docs/en/25-application/03-immigrate.md +++ b/docs/en/25-application/03-immigrate.md @@ -379,11 +379,11 @@ We still use the hypothetical environment from Chapter 4. There are three measur ### Storage resource estimation -Assuming that the number of sensor devices that generate data and need to be stored is `n`, the frequency of data generation is `t` per second, and the length of each record is `L` bytes, the scale of data generated per day is `n * t * L` bytes. Assuming the compression ratio is `C`, the daily data size is `(n * t * L)/C` bytes. The storage resources are estimated to accommodate the data scale for 1.5 years. In the production environment, the compression ratio C of TDengine is generally between 5 and 7. +Assuming that the number of sensor devices that generate data and need to be stored is `n`, the frequency of data generation is `t` per second, and the length of each record is `L` bytes, the scale of data generated per day is `86400 * n * t * L` bytes. Assuming the compression ratio is `C`, the daily data size is `(86400 * n * t * L)/C` bytes. The storage resources are estimated to accommodate the data scale for 1.5 years. In the production environment, the compression ratio C of TDengine is generally between 5 and 7. With additional 20% redundancy, you can calculate the required storage resources: ```matlab -(n * t * L) * (365 * 1.5) * (1+20%)/C +(86400 * n * t * L) * (365 * 1.5) * (1+20%)/C ```` Substituting in the above formula, the raw data generated every year is 11.8TB without considering the label information. Note that tag information is associated with each timeline in TDengine, not every record. The amount of data to be recorded is somewhat reduced relative to the generated data, and label data can be ignored as a whole. Assuming a compression ratio of 5, the size of the retained data ends up being 2.56 TB. From adb90b68e3f41cb978c640533c3d86c44c659d3c Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Fri, 8 Jul 2022 09:47:41 +0800 Subject: [PATCH 31/38] fix(stream): window_close crash --- source/libs/executor/src/timewindowoperator.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 9763f86838..f2a9a13c52 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1316,7 +1316,7 @@ bool doDeleteIntervalWindow(SAggSupporter* pAggSup, TSKEY ts, uint64_t groupId) // window has been closed return false; } - SFilePage* bufPage = getBufPage(pAggSup->pResultBuf, p1->pageId); + // SFilePage* bufPage = getBufPage(pAggSup->pResultBuf, p1->pageId); // dBufSetBufPageRecycled(pAggSup->pResultBuf, bufPage); taosHashRemove(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); return true; @@ -1419,7 +1419,7 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, ASSERT(pRecyPages != NULL); taosArrayPush(pRecyPages, &pPos->pageId); } else { - SFilePage* bufPage = getBufPage(pDiscBuf, pPos->pageId); + // SFilePage* bufPage = getBufPage(pDiscBuf, pPos->pageId); // dBufSetBufPageRecycled(pDiscBuf, bufPage); } char keyBuf[GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))]; @@ -1446,7 +1446,7 @@ static void freeAllPages(SArray* pageIds, SDiskbasedBuf* pDiskBuf) { int32_t size = taosArrayGetSize(pageIds); for (int32_t i = 0; i < size; i++) { int32_t pageId = *(int32_t*)taosArrayGet(pageIds, i); - SFilePage* bufPage = getBufPage(pDiskBuf, pageId); + // SFilePage* bufPage = getBufPage(pDiskBuf, pageId); // dBufSetBufPageRecycled(pDiskBuf, bufPage); } taosArrayClear(pageIds); From 1ee546841111b2b440366a19774a6d01f34b68cd Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 8 Jul 2022 14:47:00 +0800 Subject: [PATCH 32/38] fix(sync): append entries batch --- include/libs/sync/sync.h | 3 - source/libs/sync/src/syncAppendEntries.c | 355 +++++---------------- source/libs/sync/src/syncMain.c | 6 +- source/libs/sync/src/syncRaftLog.c | 16 + source/libs/sync/src/syncReplication.c | 15 +- source/libs/sync/test/syncLogStoreTest.cpp | 1 + tests/script/tsim/sync/vnodesnapshot.sim | 25 +- 7 files changed, 133 insertions(+), 288 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 8a6d6b7722..bef26cb310 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -163,9 +163,6 @@ typedef struct SSyncLogStore { // return commit index of log SyncIndex (*getCommitIndex)(struct SSyncLogStore* pLogStore); - // refactor, log[0 .. n] ==> log[m .. n] - // int32_t (*syncLogSetBeginIndex)(struct SSyncLogStore* pLogStore, SyncIndex beginIndex); - SyncIndex (*syncLogBeginIndex)(struct SSyncLogStore* pLogStore); SyncIndex (*syncLogEndIndex)(struct SSyncLogStore* pLogStore); bool (*syncLogIsEmpty)(struct SSyncLogStore* pLogStore); diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 2d511b6003..e43c1e4b33 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -162,6 +162,17 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { pReply->success = false; pReply->matchIndex = SYNC_INDEX_INVALID; + // msg event log + do { + char host[128]; + uint16_t port; + syncUtilU642Addr(pReply->destId.addr, host, sizeof(host), &port); + sDebug( + "vgId:%d, send sync-append-entries-reply to %s:%d, {term:%lu, pterm:%lu, success:%d, " + "match-index:%ld}", + ths->vgId, host, port, pReply->term, pReply->privateTerm, pReply->success, pReply->matchIndex); + } while (0); + SRpcMsg rpcMsg; syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); @@ -334,270 +345,16 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { pReply->matchIndex = pMsg->prevLogIndex; } - SRpcMsg rpcMsg; - syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); - syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); - syncAppendEntriesReplyDestroy(pReply); - - // maybe update commit index from leader - if (pMsg->commitIndex > ths->commitIndex) { - // has commit entry in local - if (pMsg->commitIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) { - SyncIndex beginIndex = ths->commitIndex + 1; - SyncIndex endIndex = pMsg->commitIndex; - - // update commit index - ths->commitIndex = pMsg->commitIndex; - - // call back Wal - ths->pLogStore->updateCommitIndex(ths->pLogStore, ths->commitIndex); - - int32_t code = syncNodeCommit(ths, beginIndex, endIndex, ths->state); - ASSERT(code == 0); - } - } - } - - return ret; -} - -#if 0 - -int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { - int32_t ret = 0; - - char logBuf[128] = {0}; - snprintf(logBuf, sizeof(logBuf), "==syncNodeOnAppendEntriesCb== term:%lu", ths->pRaftStore->currentTerm); - syncAppendEntriesLog2(logBuf, pMsg); - - if (pMsg->term > ths->pRaftStore->currentTerm) { - syncNodeUpdateTerm(ths, pMsg->term); - } - ASSERT(pMsg->term <= ths->pRaftStore->currentTerm); - - // reset elect timer - if (pMsg->term == ths->pRaftStore->currentTerm) { - ths->leaderCache = pMsg->srcId; - syncNodeResetElectTimer(ths); - } - ASSERT(pMsg->dataLen >= 0); - - SyncTerm localPreLogTerm = 0; - if (pMsg->prevLogIndex >= SYNC_INDEX_BEGIN && pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) { - SSyncRaftEntry* pEntry = ths->pLogStore->getEntry(ths->pLogStore, pMsg->prevLogIndex); - if (pEntry == NULL) { - char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "getEntry error, index:%ld, since %s", pMsg->prevLogIndex, terrstr()); - syncNodeErrorLog(ths, logBuf); - return -1; - } - - localPreLogTerm = pEntry->term; - syncEntryDestory(pEntry); - } - - bool logOK = - (pMsg->prevLogIndex == SYNC_INDEX_INVALID) || - ((pMsg->prevLogIndex >= SYNC_INDEX_BEGIN) && - (pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) && (pMsg->prevLogTerm == localPreLogTerm)); - - // reject request - if ((pMsg->term < ths->pRaftStore->currentTerm) || - ((pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && !logOK)) { - sTrace( - "syncNodeOnAppendEntriesCb --> reject, pMsg->term:%lu, ths->pRaftStore->currentTerm:%lu, ths->state:%d, " - "logOK:%d", - pMsg->term, ths->pRaftStore->currentTerm, ths->state, logOK); - - SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); - pReply->srcId = ths->myRaftId; - pReply->destId = pMsg->srcId; - pReply->term = ths->pRaftStore->currentTerm; - pReply->success = false; - pReply->matchIndex = SYNC_INDEX_INVALID; - - SRpcMsg rpcMsg; - syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); - syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); - syncAppendEntriesReplyDestroy(pReply); - - return ret; - } - - // return to follower state - if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_CANDIDATE) { - sTrace( - "syncNodeOnAppendEntriesCb --> return to follower, pMsg->term:%lu, ths->pRaftStore->currentTerm:%lu, " - "ths->state:%d, logOK:%d", - pMsg->term, ths->pRaftStore->currentTerm, ths->state, logOK); - - syncNodeBecomeFollower(ths, "from candidate by append entries"); - - // ret or reply? - return ret; - } - - // accept request - if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_FOLLOWER && logOK) { - // preIndex = -1, or has preIndex entry in local log - ASSERT(pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)); - - // has extra entries (> preIndex) in local log - bool hasExtraEntries = pMsg->prevLogIndex < ths->pLogStore->getLastIndex(ths->pLogStore); - - // has entries in SyncAppendEntries msg - bool hasAppendEntries = pMsg->dataLen > 0; - - sTrace( - "syncNodeOnAppendEntriesCb --> accept, pMsg->term:%lu, ths->pRaftStore->currentTerm:%lu, ths->state:%d, " - "logOK:%d, hasExtraEntries:%d, hasAppendEntries:%d", - pMsg->term, ths->pRaftStore->currentTerm, ths->state, logOK, hasExtraEntries, hasAppendEntries); - - if (hasExtraEntries && hasAppendEntries) { - // not conflict by default - bool conflict = false; - - SyncIndex extraIndex = pMsg->prevLogIndex + 1; - SSyncRaftEntry* pExtraEntry = ths->pLogStore->getEntry(ths->pLogStore, extraIndex); - if (pExtraEntry == NULL) { - char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "getEntry error2, index:%ld, since %s", extraIndex, terrstr()); - syncNodeErrorLog(ths, logBuf); - return -1; - } - - SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); - if (pAppendEntry == NULL) { - syncNodeErrorLog(ths, "syncEntryDeserialize pAppendEntry error"); - return -1; - } - - // log not match, conflict - ASSERT(extraIndex == pAppendEntry->index); - if (pExtraEntry->term != pAppendEntry->term) { - conflict = true; - } - - if (conflict) { - // roll back - SyncIndex delBegin = ths->pLogStore->getLastIndex(ths->pLogStore); - SyncIndex delEnd = extraIndex; - - sTrace("syncNodeOnAppendEntriesCb --> conflict:%d, delBegin:%ld, delEnd:%ld", conflict, delBegin, delEnd); - - // notice! reverse roll back! - for (SyncIndex index = delEnd; index >= delBegin; --index) { - if (ths->pFsm->FpRollBackCb != NULL) { - SSyncRaftEntry* pRollBackEntry = ths->pLogStore->getEntry(ths->pLogStore, index); - if (pRollBackEntry == NULL) { - char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "getEntry error3, index:%ld, since %s", index, terrstr()); - syncNodeErrorLog(ths, logBuf); - return -1; - } - - // if (pRollBackEntry->msgType != TDMT_SYNC_NOOP) { - if (syncUtilUserRollback(pRollBackEntry->msgType)) { - SRpcMsg rpcMsg; - syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg); - - SFsmCbMeta cbMeta = {0}; - cbMeta.index = pRollBackEntry->index; - cbMeta.lastConfigIndex = syncNodeGetSnapshotConfigIndex(ths, cbMeta.index); - cbMeta.isWeak = pRollBackEntry->isWeak; - cbMeta.code = 0; - cbMeta.state = ths->state; - cbMeta.seqNum = pRollBackEntry->seqNum; - ths->pFsm->FpRollBackCb(ths->pFsm, &rpcMsg, cbMeta); - rpcFreeCont(rpcMsg.pCont); - } - - syncEntryDestory(pRollBackEntry); - } - } - - // delete confict entries - ths->pLogStore->truncate(ths->pLogStore, extraIndex); - - // append new entries - ths->pLogStore->appendEntry(ths->pLogStore, pAppendEntry); - - // pre commit - SRpcMsg rpcMsg; - syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); - if (ths->pFsm != NULL) { - // if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_SYNC_NOOP) { - if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pAppendEntry->originalRpcType)) { - SFsmCbMeta cbMeta = {0}; - cbMeta.index = pAppendEntry->index; - cbMeta.lastConfigIndex = syncNodeGetSnapshotConfigIndex(ths, cbMeta.index); - cbMeta.isWeak = pAppendEntry->isWeak; - cbMeta.code = 2; - cbMeta.state = ths->state; - cbMeta.seqNum = pAppendEntry->seqNum; - ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, cbMeta); - } - } - rpcFreeCont(rpcMsg.pCont); - } - - // free memory - syncEntryDestory(pExtraEntry); - syncEntryDestory(pAppendEntry); - - } else if (hasExtraEntries && !hasAppendEntries) { - // do nothing - - } else if (!hasExtraEntries && hasAppendEntries) { - SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); - if (pAppendEntry == NULL) { - syncNodeErrorLog(ths, "syncEntryDeserialize pAppendEntry2 error"); - return -1; - } - - // append new entries - ths->pLogStore->appendEntry(ths->pLogStore, pAppendEntry); - - // pre commit - SRpcMsg rpcMsg; - syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); - if (ths->pFsm != NULL) { - // if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_SYNC_NOOP) { - if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pAppendEntry->originalRpcType)) { - SFsmCbMeta cbMeta = {0}; - cbMeta.index = pAppendEntry->index; - cbMeta.lastConfigIndex = syncNodeGetSnapshotConfigIndex(ths, cbMeta.index); - cbMeta.isWeak = pAppendEntry->isWeak; - cbMeta.code = 3; - cbMeta.state = ths->state; - cbMeta.seqNum = pAppendEntry->seqNum; - ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, cbMeta); - } - } - rpcFreeCont(rpcMsg.pCont); - - // free memory - syncEntryDestory(pAppendEntry); - - } else if (!hasExtraEntries && !hasAppendEntries) { - // do nothing - - } else { - syncNodeLog3("", ths); - ASSERT(0); - } - - SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); - pReply->srcId = ths->myRaftId; - pReply->destId = pMsg->srcId; - pReply->term = ths->pRaftStore->currentTerm; - pReply->success = true; - - if (hasAppendEntries) { - pReply->matchIndex = pMsg->prevLogIndex + 1; - } else { - pReply->matchIndex = pMsg->prevLogIndex; - } + // msg event log + do { + char host[128]; + uint16_t port; + syncUtilU642Addr(pReply->destId.addr, host, sizeof(host), &port); + sDebug( + "vgId:%d, send sync-append-entries-reply to %s:%d, {term:%lu, pterm:%lu, success:%d, " + "match-index:%ld}", + ths->vgId, host, port, pReply->term, pReply->privateTerm, pReply->success, pReply->matchIndex); + } while (0); SRpcMsg rpcMsg; syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); @@ -626,8 +383,6 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { return ret; } -#endif - static int32_t syncNodeMakeLogSame(SSyncNode* ths, SyncAppendEntries* pMsg) { int32_t code; @@ -897,6 +652,17 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc pReply->success = true; pReply->matchIndex = matchIndex; + // msg event log + do { + char host[128]; + uint16_t port; + syncUtilU642Addr(pReply->destId.addr, host, sizeof(host), &port); + sDebug( + "vgId:%d, send sync-append-entries-reply to %s:%d, {term:%lu, pterm:%lu, success:%d, " + "match-index:%ld}", + ths->vgId, host, port, pReply->term, pReply->privateTerm, pReply->success, pReply->matchIndex); + } while (0); + // send response SRpcMsg rpcMsg; syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); @@ -945,6 +711,17 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc pReply->success = false; pReply->matchIndex = SYNC_INDEX_INVALID; + // msg event log + do { + char host[128]; + uint16_t port; + syncUtilU642Addr(pReply->destId.addr, host, sizeof(host), &port); + sDebug( + "vgId:%d, send sync-append-entries-reply to %s:%d, {term:%lu, pterm:%lu, success:%d, " + "match-index:%ld}", + ths->vgId, host, port, pReply->term, pReply->privateTerm, pReply->success, pReply->matchIndex); + } while (0); + // send response SRpcMsg rpcMsg; syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); @@ -977,7 +754,7 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc do { char logBuf[128]; snprintf(logBuf, sizeof(logBuf), - "recv sync-append-entries, match, {pre-index:%ld, pre-term:%lu, datalen:%d, datacount:%d}", + "recv sync-append-entries-batch, match, {pre-index:%ld, pre-term:%lu, datalen:%d, datacount:%d}", pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen, pMsg->dataCount); syncNodeEventLog(ths, logBuf); } while (0); @@ -1018,6 +795,17 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc pReply->success = true; pReply->matchIndex = hasAppendEntries ? pMsg->prevLogIndex + pMsg->dataCount : pMsg->prevLogIndex; + // msg event log + do { + char host[128]; + uint16_t port; + syncUtilU642Addr(pReply->destId.addr, host, sizeof(host), &port); + sDebug( + "vgId:%d, send sync-append-entries-reply to %s:%d, {term:%lu, pterm:%lu, success:%d, " + "match-index:%ld}", + ths->vgId, host, port, pReply->term, pReply->privateTerm, pReply->success, pReply->matchIndex); + } while (0); + // send response SRpcMsg rpcMsg; syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); @@ -1227,6 +1015,17 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs pReply->success = true; pReply->matchIndex = matchIndex; + // msg event log + do { + char host[128]; + uint16_t port; + syncUtilU642Addr(pReply->destId.addr, host, sizeof(host), &port); + sDebug( + "vgId:%d, send sync-append-entries-reply to %s:%d, {term:%lu, pterm:%lu, success:%d, " + "match-index:%ld}", + ths->vgId, host, port, pReply->term, pReply->privateTerm, pReply->success, pReply->matchIndex); + } while (0); + // send response SRpcMsg rpcMsg; syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); @@ -1272,6 +1071,17 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs pReply->success = false; pReply->matchIndex = SYNC_INDEX_INVALID; + // msg event log + do { + char host[128]; + uint16_t port; + syncUtilU642Addr(pReply->destId.addr, host, sizeof(host), &port); + sDebug( + "vgId:%d, send sync-append-entries-reply to %s:%d, {term:%lu, pterm:%lu, success:%d, " + "match-index:%ld}", + ths->vgId, host, port, pReply->term, pReply->privateTerm, pReply->success, pReply->matchIndex); + } while (0); + // send response SRpcMsg rpcMsg; syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); @@ -1337,6 +1147,17 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs pReply->success = true; pReply->matchIndex = hasAppendEntries ? pMsg->prevLogIndex + 1 : pMsg->prevLogIndex; + // msg event log + do { + char host[128]; + uint16_t port; + syncUtilU642Addr(pReply->destId.addr, host, sizeof(host), &port); + sDebug( + "vgId:%d, send sync-append-entries-reply to %s:%d, {term:%lu, pterm:%lu, success:%d, " + "match-index:%ld}", + ths->vgId, host, port, pReply->term, pReply->privateTerm, pReply->success, pReply->matchIndex); + } while (0); + // send response SRpcMsg rpcMsg; syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 5951078822..f66313bfcd 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1311,8 +1311,10 @@ int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRp pMsg->info.noResp = 1; pSyncNode->FpSendMsg(&epSet, pMsg); } else { - sTrace("syncNodeSendMsgById pSyncNode->FpSendMsg is NULL"); + sError("vgId:%d, sync send msg by id error, fp-send-msg is null", pSyncNode->vgId); + return -1; } + return 0; } @@ -1326,7 +1328,7 @@ int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, S pMsg->info.noResp = 1; pSyncNode->FpSendMsg(&epSet, pMsg); } else { - sTrace("syncNodeSendMsgByInfo pSyncNode->FpSendMsg is NULL"); + sError("vgId:%d, sync send msg by info error, fp-send-msg is null", pSyncNode->vgId); } return 0; } diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 83495e7486..1ea07f91c3 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -326,6 +326,14 @@ static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIn ASSERT(0); } + + // event log + do { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "wal truncate, from-index:%ld", fromIndex); + syncNodeEventLog(pData->pSyncNode, logBuf); + } while (0); + return code; } @@ -463,6 +471,14 @@ int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex) { ASSERT(0); } + + // event log + do { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "wal truncate, from-index:%ld", fromIndex); + syncNodeEventLog(pData->pSyncNode, logBuf); + } while (0); + return 0; } diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 17d9aeaea0..f1c093fe37 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -151,14 +151,6 @@ int32_t syncNodeAppendEntriesPeersSnapshot2(SSyncNode* pSyncNode) { for (int32_t i = 0; i < pSyncNode->pRaftCfg->batchSize; ++i) { SSyncRaftEntry* pEntry = NULL; int32_t code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, getEntryIndex, &pEntry); - - // event log - do { - char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "get index:%d, code:%d, %s", getEntryIndex, code, tstrerror(terrno)); - syncNodeEventLog(pSyncNode, logBuf); - } while (0); - if (code == 0) { ASSERT(pEntry != NULL); entryPArr[i] = pEntry; @@ -172,8 +164,11 @@ int32_t syncNodeAppendEntriesPeersSnapshot2(SSyncNode* pSyncNode) { // event log do { - char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "build batch:%d", getCount); + char logBuf[128]; + char host[64]; + uint16_t port; + syncUtilU642Addr(pDestId->addr, host, sizeof(host), &port); + snprintf(logBuf, sizeof(logBuf), "build batch:%d for %s:%d", getCount, host, port); syncNodeEventLog(pSyncNode, logBuf); } while (0); diff --git a/source/libs/sync/test/syncLogStoreTest.cpp b/source/libs/sync/test/syncLogStoreTest.cpp index 27e1009335..9cb8194aa7 100644 --- a/source/libs/sync/test/syncLogStoreTest.cpp +++ b/source/libs/sync/test/syncLogStoreTest.cpp @@ -85,6 +85,7 @@ void logStoreTest() { } int main(int argc, char** argv) { + gRaftDetailLog = true; tsAsyncLog = 0; sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; diff --git a/tests/script/tsim/sync/vnodesnapshot.sim b/tests/script/tsim/sync/vnodesnapshot.sim index da97f40f1a..aa42047400 100644 --- a/tests/script/tsim/sync/vnodesnapshot.sim +++ b/tests/script/tsim/sync/vnodesnapshot.sim @@ -142,20 +142,33 @@ sql create table ct1 using stb tags(1000) system sh/exec.sh -n dnode4 -s stop -x SIGINT -sql insert into ct1 values(now+0s, 10, 2.0, 3.0) -sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) +$N = 100 +$count = 0 +while $count < $N + $ms = 1591200000000 + $count + sql insert into ct1 values( $ms , $count , 2.1, 3.1) + $count = $count + 1 +endw + + + +#sql insert into ct1 values(now+0s, 10, 2.0, 3.0) +#sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) #sql flush database db; #system sh/exec.sh -n dnode4 -s start -sql insert into ct1 values(now+1s, 81, 8.1, 8.1)(now+2s, -92, -9.2, -9.2)(now+3s, -73, -7.3, -7.3) +#sql insert into ct1 values(now+1s, 81, 8.1, 8.1)(now+2s, -92, -9.2, -9.2)(now+3s, -73, -7.3, -7.3) -#system sh/exec.sh -n dnode1 -s stop -x SIGINT -#system sh/exec.sh -n dnode2 -s stop -x SIGINT -#system sh/exec.sh -n dnode3 -s stop -x SIGINT +sleep 5000 + + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT #system sh/exec.sh -n dnode4 -s stop -x SIGINT From a39de34984d06b575e24a6bb4ff7b0e1a746c0f8 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 8 Jul 2022 15:21:44 +0800 Subject: [PATCH 33/38] refactor(sync): rm syncOnMessage --- source/libs/sync/inc/syncOnMessage.h | 72 ---------------------------- source/libs/sync/src/syncOnMessage.c | 56 ---------------------- 2 files changed, 128 deletions(-) delete mode 100644 source/libs/sync/inc/syncOnMessage.h delete mode 100644 source/libs/sync/src/syncOnMessage.c diff --git a/source/libs/sync/inc/syncOnMessage.h b/source/libs/sync/inc/syncOnMessage.h deleted file mode 100644 index 2f8856e652..0000000000 --- a/source/libs/sync/inc/syncOnMessage.h +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_LIBS_SYNC_ON_MESSAGE_H -#define _TD_LIBS_SYNC_ON_MESSAGE_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include -#include "taosdef.h" - -// TLA+ Spec -// Receive(m) == -// LET i == m.mdest -// j == m.msource -// IN \* Any RPC with a newer term causes the recipient to advance -// \* its term first. Responses with stale terms are ignored. -// \/ UpdateTerm(i, j, m) -// \/ /\ m.mtype = RequestVoteRequest -// /\ HandleRequestVoteRequest(i, j, m) -// \/ /\ m.mtype = RequestVoteResponse -// /\ \/ DropStaleResponse(i, j, m) -// \/ HandleRequestVoteResponse(i, j, m) -// \/ /\ m.mtype = AppendEntriesRequest -// /\ HandleAppendEntriesRequest(i, j, m) -// \/ /\ m.mtype = AppendEntriesResponse -// /\ \/ DropStaleResponse(i, j, m) -// \/ HandleAppendEntriesResponse(i, j, m) - -// DuplicateMessage(m) == -// /\ Send(m) -// /\ UNCHANGED <> - -// DropMessage(m) == -// /\ Discard(m) -// /\ UNCHANGED <> - -// Next == /\ \/ \E i \in Server : Restart(i) -// \/ \E i \in Server : Timeout(i) -// \/ \E i,j \in Server : RequestVote(i, j) -// \/ \E i \in Server : BecomeLeader(i) -// \/ \E i \in Server, v \in Value : ClientRequest(i, v) -// \/ \E i \in Server : AdvanceCommitIndex(i) -// \/ \E i,j \in Server : AppendEntries(i, j) -// \/ \E m \in DOMAIN messages : Receive(m) -// \/ \E m \in DOMAIN messages : DuplicateMessage(m) -// \/ \E m \in DOMAIN messages : DropMessage(m) -// \* History variable that tracks every log ever: -// /\ allLogs' = allLogs \cup {log[i] : i \in Server} -// - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_LIBS_SYNC_ON_MESSAGE_H*/ diff --git a/source/libs/sync/src/syncOnMessage.c b/source/libs/sync/src/syncOnMessage.c deleted file mode 100644 index ce8bed9cd3..0000000000 --- a/source/libs/sync/src/syncOnMessage.c +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "syncOnMessage.h" - -// TLA+ Spec -// Receive(m) == -// LET i == m.mdest -// j == m.msource -// IN \* Any RPC with a newer term causes the recipient to advance -// \* its term first. Responses with stale terms are ignored. -// \/ UpdateTerm(i, j, m) -// \/ /\ m.mtype = RequestVoteRequest -// /\ HandleRequestVoteRequest(i, j, m) -// \/ /\ m.mtype = RequestVoteResponse -// /\ \/ DropStaleResponse(i, j, m) -// \/ HandleRequestVoteResponse(i, j, m) -// \/ /\ m.mtype = AppendEntriesRequest -// /\ HandleAppendEntriesRequest(i, j, m) -// \/ /\ m.mtype = AppendEntriesResponse -// /\ \/ DropStaleResponse(i, j, m) -// \/ HandleAppendEntriesResponse(i, j, m) - -// DuplicateMessage(m) == -// /\ Send(m) -// /\ UNCHANGED <> - -// DropMessage(m) == -// /\ Discard(m) -// /\ UNCHANGED <> - -// Next == /\ \/ \E i \in Server : Restart(i) -// \/ \E i \in Server : Timeout(i) -// \/ \E i,j \in Server : RequestVote(i, j) -// \/ \E i \in Server : BecomeLeader(i) -// \/ \E i \in Server, v \in Value : ClientRequest(i, v) -// \/ \E i \in Server : AdvanceCommitIndex(i) -// \/ \E i,j \in Server : AppendEntries(i, j) -// \/ \E m \in DOMAIN messages : Receive(m) -// \/ \E m \in DOMAIN messages : DuplicateMessage(m) -// \/ \E m \in DOMAIN messages : DropMessage(m) -// \* History variable that tracks every log ever: -// /\ allLogs' = allLogs \cup {log[i] : i \in Server} -// \ No newline at end of file From 301edfe79b8611c3133ccbed632e2d69ffcb5740 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 8 Jul 2022 15:23:11 +0800 Subject: [PATCH 34/38] fix: set groupId for rsma and comment out rsma test case --- source/dnode/vnode/src/sma/smaRollup.c | 9 +++++++-- source/libs/executor/src/scanoperator.c | 12 +++++++----- tests/script/jenkins/basic.txt | 4 ++-- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 6f87c01be1..4c2e606e0e 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -277,8 +277,9 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo pItem->maxDelay = TSDB_MAX_ROLLUP_MAX_DELAY; } pItem->level = (idx == 0 ? TSDB_RETENTION_L1 : TSDB_RETENTION_L2); - smaInfo("vgId:%d table:%" PRIi64 " level:%" PRIi8 " maxdelay:%" PRIi64 " watermark:%" PRIi64 ", finally maxdelay:%"PRIi32, SMA_VID(pSma), - pRSmaInfo->suid, idx + 1, param->maxdelay[idx], param->watermark[idx], pItem->maxDelay); + smaInfo("vgId:%d table:%" PRIi64 " level:%" PRIi8 " maxdelay:%" PRIi64 " watermark:%" PRIi64 + ", finally maxdelay:%" PRIi32, + SMA_VID(pSma), pRSmaInfo->suid, idx + 1, param->maxdelay[idx], param->watermark[idx], pItem->maxDelay); } return TSDB_CODE_SUCCESS; _err: @@ -572,11 +573,15 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType) SSubmitReq *pReq = NULL; // TODO: the schema update should be handled if (buildSubmitReqFromDataBlock(&pReq, pResult, pRSmaInfo->pTSchema, SMA_VID(pSma), pRSmaInfo->suid) < 0) { + smaError("vgId:%d, build submit req for rsma table %" PRIi64 "l evel %" PRIi8 " failed since %s", SMA_VID(pSma), + pRSmaInfo->suid, pItem->level, terrstr()); goto _err; } if (pReq && tdProcessSubmitReq(sinkTsdb, atomic_add_fetch_64(&pRSmaInfo->pStat->submitVer, 1), pReq) < 0) { taosMemoryFreeClear(pReq); + smaError("vgId:%d, process submit req for rsma table %" PRIi64 " level %" PRIi8 " failed since %s", SMA_VID(pSma), + pRSmaInfo->suid, pItem->level, terrstr()); goto _err; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 68cda52b10..b1bad28294 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1248,11 +1248,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pInfo->pRes->info.type = STREAM_NORMAL; pInfo->pRes->info.capacity = block.info.rows; - // for generating rollup SMA result, each time is an independent time serie. - // TODO temporarily used, when the statement of "partition by tbname" is ready, remove this - if (pInfo->assignBlockUid) { - pInfo->pRes->info.groupId = block.info.uid; - } + uint64_t* groupIdPre = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &block.info.uid, sizeof(int64_t)); if (groupIdPre) { @@ -1261,6 +1257,12 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pInfo->pRes->info.groupId = 0; } + // for generating rollup SMA result, each time is an independent time serie. + // TODO temporarily used, when the statement of "partition by tbname" is ready, remove this + if (pInfo->assignBlockUid) { + pInfo->pRes->info.groupId = block.info.uid; + } + // todo extract method for (int32_t i = 0; i < taosArrayGetSize(pInfo->pColMatchInfo); ++i) { SColMatchInfo* pColMatchInfo = taosArrayGet(pInfo->pColMatchInfo, i); diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 4e009e702d..0783aa0fd1 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -163,8 +163,8 @@ # --- sma ./test.sh -f tsim/sma/drop_sma.sim ./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim -#./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim -#./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim +./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim +./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim # --- valgrind ./test.sh -f tsim/valgrind/checkError1.sim From ed6001d5cacffe87043d7964d6c346b6b2931119 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 8 Jul 2022 15:31:50 +0800 Subject: [PATCH 35/38] refactor(sync): add syncRestoreFromSnapshot --- source/libs/sync/test/CMakeLists.txt | 14 ++++ .../sync/test/syncRestoreFromSnapshot.cpp | 78 +++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 source/libs/sync/test/syncRestoreFromSnapshot.cpp diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index 37d9707cfd..e1f3a2b2fc 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -54,6 +54,7 @@ add_executable(syncRaftLogTest2 "") add_executable(syncRaftLogTest3 "") add_executable(syncLeaderTransferTest "") add_executable(syncReconfigFinishTest "") +add_executable(syncRestoreFromSnapshot "") target_sources(syncTest @@ -280,6 +281,10 @@ target_sources(syncReconfigFinishTest PRIVATE "syncReconfigFinishTest.cpp" ) +target_sources(syncRestoreFromSnapshot + PRIVATE + "syncRestoreFromSnapshot.cpp" +) target_include_directories(syncTest @@ -562,6 +567,11 @@ target_include_directories(syncReconfigFinishTest "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncRestoreFromSnapshot + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries(syncTest @@ -788,6 +798,10 @@ target_link_libraries(syncReconfigFinishTest sync gtest_main ) +target_link_libraries(syncRestoreFromSnapshot + sync + gtest_main +) enable_testing() diff --git a/source/libs/sync/test/syncRestoreFromSnapshot.cpp b/source/libs/sync/test/syncRestoreFromSnapshot.cpp new file mode 100644 index 0000000000..97f3aa8220 --- /dev/null +++ b/source/libs/sync/test/syncRestoreFromSnapshot.cpp @@ -0,0 +1,78 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" +#include "wal.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +void init() { + int code = walInit(); + assert(code == 0); +} + +void cleanup() { walCleanUp(); } + +SWal* createWal(char* path, int32_t vgId) { + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = vgId; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + SWal* pWal = walOpen(path, &walCfg); + assert(pWal != NULL); + return pWal; +} + +SSyncNode* createSyncNode(SWal* pWal) { + SSyncNode* pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(SSyncNode)); + memset(pSyncNode, 0, sizeof(SSyncNode)); + pSyncNode->pWal = pWal; + return pSyncNode; +} + +void usage(char* exe) { printf("usage: %s path vgId snapshotIndex \n", exe); } + +int main(int argc, char** argv) { + if (argc != 3) { + usage(argv[0]); + exit(-1); + } + char* path = argv[1]; + int32_t vgId = atoi(argv[2]); + int64_t snapshotIndex = atoll(argv[3]); + + init(); + SWal* pWal = createWal(path, vgId); + assert(pWal != NULL); + SSyncNode* pSyncNode = createSyncNode(pWal); + assert(pSyncNode != NULL); + + SSyncLogStore* pLog = logStoreCreate(pSyncNode); + assert(pLog != NULL); + + int32_t code = pLog->syncLogRestoreFromSnapshot(pLog, snapshotIndex); + assert(code == 0); + + walClose(pWal); + logStoreDestory(pLog); + taosMemoryFree(pSyncNode); + + cleanup(); + return 0; +} From 96db67699586736ad4e9c6f6d9456c08d0515a85 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 8 Jul 2022 15:37:45 +0800 Subject: [PATCH 36/38] refactor: delete redundant files --- source/common/src/tdata.c | 14 - source/common/src/tmsgtype.c | 18 - source/dnode/vnode/CMakeLists.txt | 3 - source/dnode/vnode/src/sma/smaSnapshot.c | 16 - source/dnode/vnode/src/tsdb/tsdbReadImpl.c | 16 - source/dnode/vnode/src/vnd/vnodeStateMgr.c | 14 - source/libs/transport/test/CMakeLists.txt | 66 +--- source/libs/transport/test/rclient.c | 216 ----------- source/libs/transport/test/rsclient.c | 196 ---------- source/libs/transport/test/rserver.c | 194 ---------- source/libs/transport/test/syncClient.c | 211 ----------- source/os/src/osStrptime.c | 405 --------------------- source/util/src/ttrace.c | 79 ---- 13 files changed, 7 insertions(+), 1441 deletions(-) delete mode 100644 source/common/src/tdata.c delete mode 100644 source/common/src/tmsgtype.c delete mode 100644 source/dnode/vnode/src/sma/smaSnapshot.c delete mode 100644 source/dnode/vnode/src/tsdb/tsdbReadImpl.c delete mode 100644 source/dnode/vnode/src/vnd/vnodeStateMgr.c delete mode 100644 source/libs/transport/test/rclient.c delete mode 100644 source/libs/transport/test/rsclient.c delete mode 100644 source/libs/transport/test/rserver.c delete mode 100644 source/libs/transport/test/syncClient.c delete mode 100644 source/os/src/osStrptime.c delete mode 100644 source/util/src/ttrace.c diff --git a/source/common/src/tdata.c b/source/common/src/tdata.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/src/tdata.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/src/tmsgtype.c b/source/common/src/tmsgtype.c deleted file mode 100644 index 3ca0f00fc2..0000000000 --- a/source/common/src/tmsgtype.c +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#define TSDB_SQL_C -#include "tmsgtype.h" diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 24c4f2912c..897e4f26e2 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -9,7 +9,6 @@ target_sources( "src/vnd/vnodeCfg.c" "src/vnd/vnodeCommit.c" "src/vnd/vnodeQuery.c" - "src/vnd/vnodeStateMgr.c" "src/vnd/vnodeModule.c" "src/vnd/vnodeSvr.c" "src/vnd/vnodeSync.c" @@ -32,7 +31,6 @@ target_sources( "src/sma/smaUtil.c" "src/sma/smaOpen.c" "src/sma/smaCommit.c" - "src/sma/smaSnapshot.c" "src/sma/smaRollup.c" "src/sma/smaTimeRange.c" @@ -43,7 +41,6 @@ target_sources( "src/tsdb/tsdbOpen.c" "src/tsdb/tsdbMemTable.c" "src/tsdb/tsdbRead.c" - "src/tsdb/tsdbReadImpl.c" "src/tsdb/tsdbCache.c" "src/tsdb/tsdbWrite.c" "src/tsdb/tsdbReaderWriter.c" diff --git a/source/dnode/vnode/src/sma/smaSnapshot.c b/source/dnode/vnode/src/sma/smaSnapshot.c deleted file mode 100644 index b2c85642b9..0000000000 --- a/source/dnode/vnode/src/sma/smaSnapshot.c +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "sma.h" \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c deleted file mode 100644 index fe0d3a1b6f..0000000000 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tsdb.h" diff --git a/source/dnode/vnode/src/vnd/vnodeStateMgr.c b/source/dnode/vnode/src/vnd/vnodeStateMgr.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/dnode/vnode/src/vnd/vnodeStateMgr.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/libs/transport/test/CMakeLists.txt b/source/libs/transport/test/CMakeLists.txt index 98a252e008..5645f49284 100644 --- a/source/libs/transport/test/CMakeLists.txt +++ b/source/libs/transport/test/CMakeLists.txt @@ -1,30 +1,16 @@ add_executable(transportTest "") -add_executable(client "") -add_executable(server "") add_executable(transUT "") -add_executable(syncClient "") add_executable(pushServer "") target_sources(transUT PRIVATE "transUT.cpp" -) +) + target_sources(transportTest PRIVATE "transportTests.cpp" -) -target_sources (client - PRIVATE - "rclient.c" -) -target_sources (server - PRIVATE - "rserver.c" -) -target_sources (syncClient - PRIVATE - "syncClient.c" -) +) target_sources(pushServer PRIVATE @@ -35,7 +21,7 @@ target_include_directories(transportTest PUBLIC "${TD_SOURCE_DIR}/include/libs/transport" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" -) +) target_link_libraries (transportTest os @@ -44,6 +30,7 @@ target_link_libraries (transportTest gtest_main transport ) + target_link_libraries (transUT os util @@ -52,56 +39,18 @@ target_link_libraries (transUT transport ) -target_include_directories(client - PUBLIC - "${TD_SOURCE_DIR}/include/libs/transport" - "${CMAKE_CURRENT_SOURCE_DIR}/../inc" -) - -target_link_libraries (client - os - util - common - gtest_main - transport -) -target_include_directories(server - PUBLIC - "${TD_SOURCE_DIR}/include/libs/transport" - "${CMAKE_CURRENT_SOURCE_DIR}/../inc" -) - target_include_directories(transUT PUBLIC "${TD_SOURCE_DIR}/include/libs/transport" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" -) - -target_link_libraries (server - os - util - common - gtest_main - transport -) -target_include_directories(syncClient - PUBLIC - "${TD_SOURCE_DIR}/include/libs/transport" - "${CMAKE_CURRENT_SOURCE_DIR}/../inc" -) -target_link_libraries (syncClient - os - util - common - gtest_main - transport ) target_include_directories(pushServer PUBLIC "${TD_SOURCE_DIR}/include/libs/transport" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" -) +) + target_link_libraries (pushServer os util @@ -110,7 +59,6 @@ target_link_libraries (pushServer transport ) - add_test( NAME transUT COMMAND transUT diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c deleted file mode 100644 index 55e6dd000a..0000000000 --- a/source/libs/transport/test/rclient.c +++ /dev/null @@ -1,216 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#include -#include "os.h" -#include "taoserror.h" -#include "tglobal.h" -#include "transLog.h" -#include "trpc.h" -#include "tutil.h" - -typedef struct { - int index; - SEpSet epSet; - int num; - int numOfReqs; - int msgSize; - tsem_t rspSem; - tsem_t * pOverSem; - TdThread thread; - void * pRpc; -} SInfo; -static void processResponse(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { - SInfo *pInfo = (SInfo *)pMsg->info.ahandle; - // tError("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, - // pMsg->code); - - if (pEpSet) pInfo->epSet = *pEpSet; - - rpcFreeCont(pMsg->pCont); - // tsem_post(&pInfo->rspSem); - tsem_post(&pInfo->rspSem); -} - -static int tcount = 0; - -static void *sendRequest(void *param) { - SInfo * pInfo = (SInfo *)param; - SRpcMsg rpcMsg = {0}; - - tError("thread:%d, start to send request", pInfo->index); - - tError("thread:%d, reqs: %d", pInfo->index, pInfo->numOfReqs); - int u100 = 0; - int u500 = 0; - int u1000 = 0; - int u10000 = 0; - - while (pInfo->numOfReqs == 0 || pInfo->num < pInfo->numOfReqs) { - pInfo->num++; - rpcMsg.pCont = rpcMallocCont(pInfo->msgSize); - rpcMsg.contLen = pInfo->msgSize; - rpcMsg.info.ahandle = pInfo; - rpcMsg.msgType = 1; - // tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num); - int64_t start = taosGetTimestampUs(); - rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg, NULL); - if (pInfo->num % 20000 == 0) tError("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); - // tsem_wait(&pInfo->rspSem); - tsem_wait(&pInfo->rspSem); - int64_t end = taosGetTimestampUs() - start; - if (end <= 100) { - u100++; - } else if (end > 100 && end <= 500) { - u500++; - } else if (end > 500 && end < 1000) { - u1000++; - } else { - u10000++; - } - - tDebug("recv response succefully"); - - // taosSsleep(100); - } - - tError("send and recv sum: %d, %d, %d, %d", u100, u500, u1000, u10000); - tError("thread:%d, it is over", pInfo->index); - tcount++; - - return NULL; -} - -int main(int argc, char *argv[]) { - SRpcInit rpcInit; - SEpSet epSet = {0}; - int msgSize = 128; - int numOfReqs = 0; - int appThreads = 1; - char serverIp[40] = "127.0.0.1"; - char secret[20] = "mypassword"; - struct timeval systemTime; - int64_t startTime, endTime; - TdThreadAttr thattr; - - // server info - epSet.inUse = 0; - addEpIntoEpSet(&epSet, serverIp, 7000); - addEpIntoEpSet(&epSet, "192.168.0.1", 7000); - - // client info - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = 0; - rpcInit.label = "APP"; - rpcInit.numOfThreads = 1; - rpcInit.cfp = processResponse; - rpcInit.sessions = 100; - rpcInit.idleTime = 100; - rpcInit.user = "michael"; - rpcInit.connType = TAOS_CONN_CLIENT; - rpcDebugFlag = 131; - - for (int i = 1; i < argc; ++i) { - if (strcmp(argv[i], "-p") == 0 && i < argc - 1) { - epSet.eps[0].port = atoi(argv[++i]); - } else if (strcmp(argv[i], "-i") == 0 && i < argc - 1) { - tstrncpy(epSet.eps[0].fqdn, argv[++i], sizeof(epSet.eps[0].fqdn)); - } else if (strcmp(argv[i], "-t") == 0 && i < argc - 1) { - rpcInit.numOfThreads = atoi(argv[++i]); - } else if (strcmp(argv[i], "-m") == 0 && i < argc - 1) { - msgSize = atoi(argv[++i]); - } else if (strcmp(argv[i], "-s") == 0 && i < argc - 1) { - rpcInit.sessions = atoi(argv[++i]); - } else if (strcmp(argv[i], "-n") == 0 && i < argc - 1) { - numOfReqs = atoi(argv[++i]); - } else if (strcmp(argv[i], "-a") == 0 && i < argc - 1) { - appThreads = atoi(argv[++i]); - } else if (strcmp(argv[i], "-o") == 0 && i < argc - 1) { - tsCompressMsgSize = atoi(argv[++i]); - } else if (strcmp(argv[i], "-u") == 0 && i < argc - 1) { - rpcInit.user = argv[++i]; - } else if (strcmp(argv[i], "-k") == 0 && i < argc - 1) { - } else if (strcmp(argv[i], "-spi") == 0 && i < argc - 1) { - } else if (strcmp(argv[i], "-d") == 0 && i < argc - 1) { - rpcDebugFlag = atoi(argv[++i]); - } else { - printf("\nusage: %s [options] \n", argv[0]); - printf(" [-i ip]: first server IP address, default is:%s\n", serverIp); - printf(" [-p port]: server port number, default is:%d\n", epSet.eps[0].port); - printf(" [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads); - printf(" [-s sessions]: number of rpc sessions, default is:%d\n", rpcInit.sessions); - printf(" [-m msgSize]: message body size, default is:%d\n", msgSize); - printf(" [-a threads]: number of app threads, default is:%d\n", appThreads); - printf(" [-n requests]: number of requests per thread, default is:%d\n", numOfReqs); - printf(" [-o compSize]: compression message size, default is:%d\n", tsCompressMsgSize); - printf(" [-u user]: user name for the connection, default is:%s\n", rpcInit.user); - printf(" [-d debugFlag]: debug flag, default:%d\n", rpcDebugFlag); - printf(" [-h help]: print out this help\n\n"); - exit(0); - } - } - - const char *path = TD_TMP_DIR_PATH "transport/client"; - taosRemoveDir(path); - taosMkDir(path); - tstrncpy(tsLogDir, path, PATH_MAX); - taosInitLog("client.log", 10); - - void *pRpc = rpcOpen(&rpcInit); - if (pRpc == NULL) { - tError("failed to initialize RPC"); - return -1; - } - - tError("client is initialized"); - tError("threads:%d msgSize:%d requests:%d", appThreads, msgSize, numOfReqs); - - taosGetTimeOfDay(&systemTime); - startTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; - - SInfo *pInfo = (SInfo *)taosMemoryCalloc(1, sizeof(SInfo) * appThreads); - - taosThreadAttrInit(&thattr); - taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); - - for (int i = 0; i < appThreads; ++i) { - pInfo->index = i; - pInfo->epSet = epSet; - pInfo->numOfReqs = numOfReqs; - pInfo->msgSize = msgSize; - tsem_init(&pInfo->rspSem, 0, 0); - pInfo->pRpc = pRpc; - taosThreadCreate(&pInfo->thread, &thattr, sendRequest, pInfo); - pInfo++; - } - - do { - taosUsleep(1); - } while (tcount < appThreads); - - taosGetTimeOfDay(&systemTime); - endTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; - float usedTime = (endTime - startTime) / 1000.0f; // mseconds - - tError("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs * appThreads); - tError("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0 * numOfReqs * appThreads / usedTime, - msgSize); - - int ch = getchar(); - UNUSED(ch); - - taosCloseLog(); - - return 0; -} diff --git a/source/libs/transport/test/rsclient.c b/source/libs/transport/test/rsclient.c deleted file mode 100644 index e7fc7b45e1..0000000000 --- a/source/libs/transport/test/rsclient.c +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - - -#include "os.h" -#include "tutil.h" -#include "tglobal.h" -#include "rpcLog.h" -#include "trpc.h" -#include "taoserror.h" - -typedef struct { - int index; - SRpcEpSet epSet; - int num; - int numOfReqs; - int msgSize; - tsem_t rspSem; - tsem_t *pOverSem; - TdThread thread; - void *pRpc; -} SInfo; - - -static int tcount = 0; -static int terror = 0; - -static void *sendRequest(void *param) { - SInfo *pInfo = (SInfo *)param; - SRpcMsg rpcMsg, rspMsg; - - tDebug("thread:%d, start to send request", pInfo->index); - - while ( pInfo->numOfReqs == 0 || pInfo->num < pInfo->numOfReqs) { - pInfo->num++; - rpcMsg.pCont = rpcMallocCont(pInfo->msgSize); - rpcMsg.contLen = pInfo->msgSize; - rpcMsg.handle = pInfo; - rpcMsg.msgType = 1; - tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num); - - rpcSendRecv(pInfo->pRpc, &pInfo->epSet, &rpcMsg, &rspMsg); - - // handle response - if (rspMsg.code != 0) terror++; - - tDebug("thread:%d, rspLen:%d code:%d", pInfo->index, rspMsg.contLen, rspMsg.code); - - rpcFreeCont(rspMsg.pCont); - - if ( pInfo->num % 20000 == 0 ) - tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); - } - - tDebug("thread:%d, it is over", pInfo->index); - tcount++; - - return NULL; -} - -int main(int argc, char *argv[]) { - SRpcInit rpcInit; - SRpcEpSet epSet; - int msgSize = 128; - int numOfReqs = 0; - int appThreads = 1; - char serverIp[40] = "127.0.0.1"; - char secret[TSDB_KEY_LEN] = "mypassword"; - struct timeval systemTime; - int64_t startTime, endTime; - TdThreadAttr thattr; - - // server info - epSet.numOfEps = 1; - epSet.inUse = 0; - epSet.port[0] = 7000; - epSet.port[1] = 7000; - strcpy(epSet.fqdn[0], serverIp); - strcpy(epSet.fqdn[1], "192.168.0.1"); - - // client info - memset(&rpcInit, 0, sizeof(rpcInit)); - //rpcInit.localIp = "0.0.0.0"; - rpcInit.localPort = 0; - rpcInit.label = "APP"; - rpcInit.numOfThreads = 1; - rpcInit.sessions = 100; - rpcInit.idleTime = 3000; //tsShellActivityTimer*1000; - rpcInit.user = "michael"; - rpcInit.secret = secret; - rpcInit.ckey = "key"; - rpcInit.spi = 1; - rpcInit.connType = TAOS_CONN_CLIENT; - - for (int i=1; iindex = i; - pInfo->epSet = epSet; - pInfo->numOfReqs = numOfReqs; - pInfo->msgSize = msgSize; - tsem_init(&pInfo->rspSem, 0, 0); - pInfo->pRpc = pRpc; - taosThreadCreate(&pInfo->thread, &thattr, sendRequest, pInfo); - pInfo++; - } - - do { - taosUsleep(1); - } while ( tcount < appThreads); - - taosGetTimeOfDay(&systemTime); - endTime = systemTime.tv_sec*1000000 + systemTime.tv_usec; - float usedTime = (endTime - startTime)/1000.0; // mseconds - - tInfo("it takes %.3f mseconds to send %d requests to server, error num:%d", usedTime, numOfReqs*appThreads, terror); - tInfo("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0*numOfReqs*appThreads/usedTime, msgSize); - - taosCloseLog(); - - return 0; -} - - diff --git a/source/libs/transport/test/rserver.c b/source/libs/transport/test/rserver.c deleted file mode 100644 index 1fd78be77d..0000000000 --- a/source/libs/transport/test/rserver.c +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -//#define _DEFAULT_SOURCE -#include "os.h" -#include "tglobal.h" -#include "tqueue.h" -#include "transLog.h" -#include "trpc.h" - -int msgSize = 128; -int commit = 0; -TdFilePtr pDataFile = NULL; -STaosQueue *qhandle = NULL; -STaosQset * qset = NULL; - -void processShellMsg() { - static int num = 0; - STaosQall *qall; - SRpcMsg * pRpcMsg, rpcMsg; - int type; - void * pvnode; - - qall = taosAllocateQall(); - - while (1) { - int numOfMsgs = taosReadAllQitemsFromQset(qset, qall, &pvnode, NULL); - tDebug("%d shell msgs are received", numOfMsgs); - if (numOfMsgs <= 0) break; - - for (int i = 0; i < numOfMsgs; ++i) { - taosGetQitem(qall, (void **)&pRpcMsg); - - if (pDataFile != NULL) { - if (taosWriteFile(pDataFile, pRpcMsg->pCont, pRpcMsg->contLen) < 0) { - tInfo("failed to write data file, reason:%s", strerror(errno)); - } - } - } - - if (commit >= 2) { - num += numOfMsgs; - // if (taosFsync(pDataFile) < 0) { - // tInfo("failed to flush data to file, reason:%s", strerror(errno)); - //} - - if (num % 10000 == 0) { - tInfo("%d request have been written into disk", num); - } - } - - taosResetQitems(qall); - for (int i = 0; i < numOfMsgs; ++i) { - taosGetQitem(qall, (void **)&pRpcMsg); - rpcFreeCont(pRpcMsg->pCont); - - memset(&rpcMsg, 0, sizeof(rpcMsg)); - rpcMsg.pCont = rpcMallocCont(msgSize); - rpcMsg.contLen = msgSize; - rpcMsg.info = pRpcMsg->info; - rpcMsg.code = 0; - rpcSendResponse(&rpcMsg); - - taosFreeQitem(pRpcMsg); - } - } - - taosFreeQall(qall); -} - -int retrieveAuthInfo(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey) { - // app shall retrieve the auth info based on meterID from DB or a data file - // demo code here only for simple demo - int ret = 0; - - if (strcmp(meterId, "michael") == 0) { - *spi = 1; - *encrypt = 0; - strcpy(secret, "mypassword"); - strcpy(ckey, "key"); - } else if (strcmp(meterId, "jeff") == 0) { - *spi = 0; - *encrypt = 0; - } else { - ret = -1; // user not there - } - - return ret; -} - -void processRequestMsg(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { - SRpcMsg *pTemp; - - pTemp = taosAllocateQitem(sizeof(SRpcMsg), DEF_QITEM); - memcpy(pTemp, pMsg, sizeof(SRpcMsg)); - - tDebug("request is received, type:%d, contLen:%d, item:%p", pMsg->msgType, pMsg->contLen, pTemp); - taosWriteQitem(qhandle, pTemp); -} - -int main(int argc, char *argv[]) { - SRpcInit rpcInit; - char dataName[20] = "server.data"; - - taosBlockSIGPIPE(); - - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = 7000; - rpcInit.label = "SER"; - rpcInit.numOfThreads = 1; - rpcInit.cfp = processRequestMsg; - rpcInit.sessions = 1000; - rpcInit.idleTime = 2 * 1500; - - rpcDebugFlag = 131; - - for (int i = 1; i < argc; ++i) { - if (strcmp(argv[i], "-p") == 0 && i < argc - 1) { - rpcInit.localPort = atoi(argv[++i]); - } else if (strcmp(argv[i], "-t") == 0 && i < argc - 1) { - rpcInit.numOfThreads = atoi(argv[++i]); - } else if (strcmp(argv[i], "-m") == 0 && i < argc - 1) { - msgSize = atoi(argv[++i]); - } else if (strcmp(argv[i], "-s") == 0 && i < argc - 1) { - rpcInit.sessions = atoi(argv[++i]); - } else if (strcmp(argv[i], "-o") == 0 && i < argc - 1) { - tsCompressMsgSize = atoi(argv[++i]); - } else if (strcmp(argv[i], "-w") == 0 && i < argc - 1) { - commit = atoi(argv[++i]); - } else if (strcmp(argv[i], "-d") == 0 && i < argc - 1) { - rpcDebugFlag = atoi(argv[++i]); - dDebugFlag = rpcDebugFlag; - uDebugFlag = rpcDebugFlag; - } else { - printf("\nusage: %s [options] \n", argv[0]); - printf(" [-p port]: server port number, default is:%d\n", rpcInit.localPort); - printf(" [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads); - printf(" [-s sessions]: number of sessions, default is:%d\n", rpcInit.sessions); - printf(" [-m msgSize]: message body size, default is:%d\n", msgSize); - printf(" [-o compSize]: compression message size, default is:%d\n", tsCompressMsgSize); - printf(" [-w write]: write received data to file(0, 1, 2), default is:%d\n", commit); - printf(" [-d debugFlag]: debug flag, default:%d\n", rpcDebugFlag); - printf(" [-h help]: print out this help\n\n"); - exit(0); - } - } - - tsAsyncLog = 0; - rpcInit.connType = TAOS_CONN_SERVER; - - const char *path = TD_TMP_DIR_PATH "transport/server"; - taosRemoveDir(path); - taosMkDir(path); - tstrncpy(tsLogDir, path, PATH_MAX); - taosInitLog("server.log", 10); - - void *pRpc = rpcOpen(&rpcInit); - if (pRpc == NULL) { - tError("failed to start RPC server"); - return -1; - } - // sleep(5); - - tInfo("RPC server is running, ctrl-c to exit"); - - if (commit) { - pDataFile = taosOpenFile(dataName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); - if (pDataFile == NULL) tInfo("failed to open data file, reason:%s", strerror(errno)); - } - qhandle = taosOpenQueue(); - qset = taosOpenQset(); - taosAddIntoQset(qset, qhandle, NULL); - - processShellMsg(); - - if (pDataFile != NULL) { - taosCloseFile(&pDataFile); - taosRemoveFile(dataName); - } - - return 0; -} diff --git a/source/libs/transport/test/syncClient.c b/source/libs/transport/test/syncClient.c deleted file mode 100644 index bc6461eaed..0000000000 --- a/source/libs/transport/test/syncClient.c +++ /dev/null @@ -1,211 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#include -#include "os.h" -#include "taoserror.h" -#include "tglobal.h" -#include "transLog.h" -#include "trpc.h" -#include "tutil.h" - -typedef struct { - int index; - SEpSet epSet; - int num; - int numOfReqs; - int msgSize; - tsem_t rspSem; - tsem_t * pOverSem; - TdThread thread; - void * pRpc; -} SInfo; -static void processResponse(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { - SInfo *pInfo = (SInfo *)pMsg->info.ahandle; - tDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, - pMsg->code); - - if (pEpSet) pInfo->epSet = *pEpSet; - - rpcFreeCont(pMsg->pCont); - // tsem_post(&pInfo->rspSem); - tsem_post(&pInfo->rspSem); -} - -static int tcount = 0; - -static void *sendRequest(void *param) { - SInfo * pInfo = (SInfo *)param; - SRpcMsg rpcMsg = {0}; - - tDebug("thread:%d, start to send request", pInfo->index); - - tDebug("thread:%d, reqs: %d", pInfo->index, pInfo->numOfReqs); - int u100 = 0; - int u500 = 0; - int u1000 = 0; - int u10000 = 0; - SRpcMsg respMsg = {0}; - while (pInfo->numOfReqs == 0 || pInfo->num < pInfo->numOfReqs) { - pInfo->num++; - rpcMsg.pCont = rpcMallocCont(pInfo->msgSize); - rpcMsg.contLen = pInfo->msgSize; - rpcMsg.info.ahandle = pInfo; - rpcMsg.msgType = 1; - // tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num); - int64_t start = taosGetTimestampUs(); - rpcSendRecv(pInfo->pRpc, &pInfo->epSet, &rpcMsg, &respMsg); - // rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg, NULL); - if (pInfo->num % 20000 == 0) tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); - // tsem_wait(&pInfo->rspSem); - // wtsem_wait(&pInfo->rspSem); - int64_t end = taosGetTimestampUs() - start; - if (end <= 100) { - u100++; - } else if (end > 100 && end <= 500) { - u500++; - } else if (end > 500 && end < 1000) { - u1000++; - } else { - u10000++; - } - - tDebug("recv response succefully"); - - // taosSsleep(100); - } - - tError("send and recv sum: %d, %d, %d, %d", u100, u500, u1000, u10000); - tDebug("thread:%d, it is over", pInfo->index); - tcount++; - - return NULL; -} - -int main(int argc, char *argv[]) { - SRpcInit rpcInit; - SEpSet epSet = {0}; - int msgSize = 128; - int numOfReqs = 0; - int appThreads = 1; - char serverIp[40] = "127.0.0.1"; - char secret[20] = "mypassword"; - struct timeval systemTime; - int64_t startTime, endTime; - TdThreadAttr thattr; - - // server info - epSet.inUse = 0; - addEpIntoEpSet(&epSet, serverIp, 7000); - addEpIntoEpSet(&epSet, "192.168.0.1", 7000); - - // client info - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = 0; - rpcInit.label = "APP"; - rpcInit.numOfThreads = 1; - rpcInit.cfp = processResponse; - rpcInit.sessions = 100; - rpcInit.idleTime = 100; - rpcInit.user = "michael"; - rpcInit.connType = TAOS_CONN_CLIENT; - - for (int i = 1; i < argc; ++i) { - if (strcmp(argv[i], "-p") == 0 && i < argc - 1) { - epSet.eps[0].port = atoi(argv[++i]); - } else if (strcmp(argv[i], "-i") == 0 && i < argc - 1) { - tstrncpy(epSet.eps[0].fqdn, argv[++i], sizeof(epSet.eps[0].fqdn)); - } else if (strcmp(argv[i], "-t") == 0 && i < argc - 1) { - rpcInit.numOfThreads = atoi(argv[++i]); - } else if (strcmp(argv[i], "-m") == 0 && i < argc - 1) { - msgSize = atoi(argv[++i]); - } else if (strcmp(argv[i], "-s") == 0 && i < argc - 1) { - rpcInit.sessions = atoi(argv[++i]); - } else if (strcmp(argv[i], "-n") == 0 && i < argc - 1) { - numOfReqs = atoi(argv[++i]); - } else if (strcmp(argv[i], "-a") == 0 && i < argc - 1) { - appThreads = atoi(argv[++i]); - } else if (strcmp(argv[i], "-o") == 0 && i < argc - 1) { - tsCompressMsgSize = atoi(argv[++i]); - } else if (strcmp(argv[i], "-u") == 0 && i < argc - 1) { - rpcInit.user = argv[++i]; - } else if (strcmp(argv[i], "-k") == 0 && i < argc - 1) { - } else if (strcmp(argv[i], "-spi") == 0 && i < argc - 1) { - } else if (strcmp(argv[i], "-d") == 0 && i < argc - 1) { - rpcDebugFlag = atoi(argv[++i]); - } else { - printf("\nusage: %s [options] \n", argv[0]); - printf(" [-i ip]: first server IP address, default is:%s\n", serverIp); - printf(" [-p port]: server port number, default is:%d\n", epSet.eps[0].port); - printf(" [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads); - printf(" [-s sessions]: number of rpc sessions, default is:%d\n", rpcInit.sessions); - printf(" [-m msgSize]: message body size, default is:%d\n", msgSize); - printf(" [-a threads]: number of app threads, default is:%d\n", appThreads); - printf(" [-n requests]: number of requests per thread, default is:%d\n", numOfReqs); - printf(" [-o compSize]: compression message size, default is:%d\n", tsCompressMsgSize); - printf(" [-u user]: user name for the connection, default is:%s\n", rpcInit.user); - printf(" [-d debugFlag]: debug flag, default:%d\n", rpcDebugFlag); - printf(" [-h help]: print out this help\n\n"); - exit(0); - } - } - - taosInitLog("client.log", 10); - - void *pRpc = rpcOpen(&rpcInit); - if (pRpc == NULL) { - tError("failed to initialize RPC"); - return -1; - } - - tInfo("client is initialized"); - tInfo("threads:%d msgSize:%d requests:%d", appThreads, msgSize, numOfReqs); - - taosGetTimeOfDay(&systemTime); - startTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; - - SInfo *pInfo = (SInfo *)taosMemoryCalloc(1, sizeof(SInfo) * appThreads); - - taosThreadAttrInit(&thattr); - taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); - - for (int i = 0; i < appThreads; ++i) { - pInfo->index = i; - pInfo->epSet = epSet; - pInfo->numOfReqs = numOfReqs; - pInfo->msgSize = msgSize; - tsem_init(&pInfo->rspSem, 0, 0); - pInfo->pRpc = pRpc; - taosThreadCreate(&pInfo->thread, &thattr, sendRequest, pInfo); - pInfo++; - } - - do { - taosUsleep(1); - } while (tcount < appThreads); - - taosGetTimeOfDay(&systemTime); - endTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; - float usedTime = (endTime - startTime) / 1000.0f; // mseconds - - tInfo("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs * appThreads); - tInfo("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0 * numOfReqs * appThreads / usedTime, msgSize); - - int ch = getchar(); - UNUSED(ch); - - taosCloseLog(); - - return 0; -} diff --git a/source/os/src/osStrptime.c b/source/os/src/osStrptime.c deleted file mode 100644 index 8d878577ea..0000000000 --- a/source/os/src/osStrptime.c +++ /dev/null @@ -1,405 +0,0 @@ -/* $Id$ */ -/* $NetBSD: strptime.c,v 1.18 1999/04/29 02:58:30 tv Exp $ */ - -/*- -* Copyright (c) 1997, 1998 The NetBSD Foundation, Inc. -* All rights reserved. -* -* This code was contributed to The NetBSD Foundation by Klaus Klein. -* -* Redistribution and use in source and binary forms, with or without -* modification, are permitted provided that the following conditions -* are met: -* 1. Redistributions of source code must retain the above copyright -* notice, this list of conditions and the following disclaimer. -* 2. Redistributions in binary form must reproduce the above copyright -* notice, this list of conditions and the following disclaimer in the -* documentation and/or other materials provided with the distribution. -* 3. All advertising materials mentioning features or use of this software -* must display the following acknowledgement: -* This product includes software developed by the NetBSD -* Foundation, Inc. and its contributors. -* 4. Neither the name of The NetBSD Foundation nor the names of its -* contributors may be used to endorse or promote products derived -* from this software without specific prior written permission. -* -* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS -* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED -* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS -* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -* POSSIBILITY OF SUCH DAMAGE. -*/ -// -//#include "lukemftp.h" - -// #ifdef WINDOWS - -// #include -// #include -// #include -// #include -// //#define TM_YEAR_BASE 1970 //origin -// #define TM_YEAR_BASE 1900 //slguan -// /* -// * We do not implement alternate representations. However, we always -// * check whether a given modifier is allowed for a certain conversion. -// */ -// #define ALT_E 0x01 -// #define ALT_O 0x02 -// #define LEGAL_ALT(x) { if (alt_format & ~(x)) return (0); } - - -// static int conv_num(const char **buf, int *dest, int llim, int ulim) -// { -// int result = 0; - -// /* The limit also determines the number of valid digits. */ -// int rulim = ulim; - -// if (**buf < '0' || **buf > '9') -// return (0); - -// do { -// result *= 10; -// result += *(*buf)++ - '0'; -// rulim /= 10; -// } while ((result * 10 <= ulim) && rulim && **buf >= '0' && **buf <= '9'); - -// if (result < llim || result > ulim) -// return (0); - -// *dest = result; -// return (1); -// } - -// static const char *day[7] = { -// "Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", -// "Friday", "Saturday" -// }; -// static const char *abday[7] = { -// "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" -// }; -// static const char *mon[12] = { -// "January", "February", "March", "April", "May", "June", "July", -// "August", "September", "October", "November", "December" -// }; -// static const char *abmon[12] = { -// "Jan", "Feb", "Mar", "Apr", "May", "Jun", -// "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" -// }; -// static const char *am_pm[2] = { -// "AM", "PM" -// }; - -// #endif - -// char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm) { -// #ifdef WINDOWS -// char c; -// const char *bp; -// size_t len = 0; -// int alt_format, i, split_year = 0; - -// bp = buf; - -// while ((c = *fmt) != '\0') { -// /* Clear `alternate' modifier prior to new conversion. */ -// alt_format = 0; - -// /* Eat up white-space. */ -// if (isspace(c)) { -// while (isspace(*bp)) -// bp++; - -// fmt++; -// continue; -// } - -// if ((c = *fmt++) != '%') -// goto literal; - - -// again: switch (c = *fmt++) { -// case '%': /* "%%" is converted to "%". */ -// literal : -// if (c != *bp++) -// return (0); -// break; - -// /* -// * "Alternative" modifiers. Just set the appropriate flag -// * and start over again. -// */ -// case 'E': /* "%E?" alternative conversion modifier. */ -// LEGAL_ALT(0); -// alt_format |= ALT_E; -// goto again; - -// case 'O': /* "%O?" alternative conversion modifier. */ -// LEGAL_ALT(0); -// alt_format |= ALT_O; -// goto again; - -// /* -// * "Complex" conversion rules, implemented through recursion. -// */ -// case 'c': /* Date and time, using the locale's format. */ -// LEGAL_ALT(ALT_E); -// if (!(bp = taosStrpTime(bp, "%x %X", tm))) -// return (0); -// break; - -// case 'D': /* The date as "%m/%d/%y". */ -// LEGAL_ALT(0); -// if (!(bp = taosStrpTime(bp, "%m/%d/%y", tm))) -// return (0); -// break; - -// case 'R': /* The time as "%H:%M". */ -// LEGAL_ALT(0); -// if (!(bp = taosStrpTime(bp, "%H:%M", tm))) -// return (0); -// break; - -// case 'r': /* The time in 12-hour clock representation. */ -// LEGAL_ALT(0); -// if (!(bp = taosStrpTime(bp, "%I:%M:%S %p", tm))) -// return (0); -// break; - -// case 'T': /* The time as "%H:%M:%S". */ -// LEGAL_ALT(0); -// if (!(bp = taosStrpTime(bp, "%H:%M:%S", tm))) -// return (0); -// break; - -// case 'X': /* The time, using the locale's format. */ -// LEGAL_ALT(ALT_E); -// if (!(bp = taosStrpTime(bp, "%H:%M:%S", tm))) -// return (0); -// break; - -// case 'x': /* The date, using the locale's format. */ -// LEGAL_ALT(ALT_E); -// if (!(bp = taosStrpTime(bp, "%m/%d/%y", tm))) -// return (0); -// break; - -// /* -// * "Elementary" conversion rules. -// */ -// case 'A': /* The day of week, using the locale's form. */ -// case 'a': -// LEGAL_ALT(0); -// for (i = 0; i < 7; i++) { -// /* Full name. */ -// len = strlen(day[i]); -// if (strncmp(day[i], bp, len) == 0) -// break; - -// /* Abbreviated name. */ -// len = strlen(abday[i]); -// if (strncmp(abday[i], bp, len) == 0) -// break; -// } - -// /* Nothing matched. */ -// if (i == 7) -// return (0); - -// tm->tm_wday = i; -// bp += len; -// break; - -// case 'B': /* The month, using the locale's form. */ -// case 'b': -// case 'h': -// LEGAL_ALT(0); -// for (i = 0; i < 12; i++) { -// /* Full name. */ -// len = strlen(mon[i]); -// if (strncmp(mon[i], bp, len) == 0) -// break; - -// /* Abbreviated name. */ -// len = strlen(abmon[i]); -// if (strncmp(abmon[i], bp, len) == 0) -// break; -// } - -// /* Nothing matched. */ -// if (i == 12) -// return (0); - -// tm->tm_mon = i; -// bp += len; -// break; - -// case 'C': /* The century number. */ -// LEGAL_ALT(ALT_E); -// if (!(conv_num(&bp, &i, 0, 99))) -// return (0); - -// if (split_year) { -// tm->tm_year = (tm->tm_year % 100) + (i * 100); -// } -// else { -// tm->tm_year = i * 100; -// split_year = 1; -// } -// break; - -// case 'd': /* The day of month. */ -// case 'e': -// LEGAL_ALT(ALT_O); -// if (!(conv_num(&bp, &tm->tm_mday, 1, 31))) -// return (0); -// break; - -// case 'k': /* The hour (24-hour clock representation). */ -// LEGAL_ALT(0); -// /* FALLTHROUGH */ -// case 'H': -// LEGAL_ALT(ALT_O); -// if (!(conv_num(&bp, &tm->tm_hour, 0, 23))) -// return (0); -// break; - -// case 'l': /* The hour (12-hour clock representation). */ -// LEGAL_ALT(0); -// /* FALLTHROUGH */ -// case 'I': -// LEGAL_ALT(ALT_O); -// if (!(conv_num(&bp, &tm->tm_hour, 1, 12))) -// return (0); -// if (tm->tm_hour == 12) -// tm->tm_hour = 0; -// break; - -// case 'j': /* The day of year. */ -// LEGAL_ALT(0); -// if (!(conv_num(&bp, &i, 1, 366))) -// return (0); -// tm->tm_yday = i - 1; -// break; - -// case 'M': /* The minute. */ -// LEGAL_ALT(ALT_O); -// if (!(conv_num(&bp, &tm->tm_min, 0, 59))) -// return (0); -// break; - -// case 'm': /* The month. */ -// LEGAL_ALT(ALT_O); -// if (!(conv_num(&bp, &i, 1, 12))) -// return (0); -// tm->tm_mon = i - 1; -// break; - -// case 'p': /* The locale's equivalent of AM/PM. */ -// LEGAL_ALT(0); -// /* AM? */ -// if (strcmp(am_pm[0], bp) == 0) { -// if (tm->tm_hour > 11) -// return (0); - -// bp += strlen(am_pm[0]); -// break; -// } -// /* PM? */ -// else if (strcmp(am_pm[1], bp) == 0) { -// if (tm->tm_hour > 11) -// return (0); - -// tm->tm_hour += 12; -// bp += strlen(am_pm[1]); -// break; -// } - -// /* Nothing matched. */ -// return (0); - -// case 'S': /* The seconds. */ -// LEGAL_ALT(ALT_O); -// if (!(conv_num(&bp, &tm->tm_sec, 0, 61))) -// return (0); -// break; - -// case 'U': /* The week of year, beginning on sunday. */ -// case 'W': /* The week of year, beginning on monday. */ -// LEGAL_ALT(ALT_O); -// /* -// * XXX This is bogus, as we can not assume any valid -// * information present in the tm structure at this -// * point to calculate a real value, so just check the -// * range for now. -// */ -// if (!(conv_num(&bp, &i, 0, 53))) -// return (0); -// break; - -// case 'w': /* The day of week, beginning on sunday. */ -// LEGAL_ALT(ALT_O); -// if (!(conv_num(&bp, &tm->tm_wday, 0, 6))) -// return (0); -// break; - -// case 'Y': /* The year. */ -// LEGAL_ALT(ALT_E); -// if (!(conv_num(&bp, &i, 0, 9999))) -// return (0); - -// tm->tm_year = i - TM_YEAR_BASE; -// break; - -// case 'y': /* The year within 100 years of the epoch. */ -// LEGAL_ALT(ALT_E | ALT_O); -// if (!(conv_num(&bp, &i, 0, 99))) -// return (0); - -// if (split_year) { -// tm->tm_year = ((tm->tm_year / 100) * 100) + i; -// break; -// } -// split_year = 1; -// if (i <= 68) -// tm->tm_year = i + 2000 - TM_YEAR_BASE; -// else -// tm->tm_year = i + 1900 - TM_YEAR_BASE; -// break; - -// /* -// * Miscellaneous conversions. -// */ -// case 'n': /* Any kind of white-space. */ -// case 't': -// LEGAL_ALT(0); -// while (isspace(*bp)) -// bp++; -// break; - - -// default: /* Unknown/unsupported conversion. */ -// return (0); -// } - - -// } - -// /* LINTED functional specification */ -// return ((char *)bp); -// #elif defined(_TD_DARWIN_64) -// return strptime(buf, fmt, tm); -// #else -// return strptime(buf, fmt, tm); -// #endif -// } - - - diff --git a/source/util/src/ttrace.c b/source/util/src/ttrace.c deleted file mode 100644 index f183fd58fd..0000000000 --- a/source/util/src/ttrace.c +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#include "ttrace.h" -#include "taos.h" -#include "thash.h" -#include "tuuid.h" - -// clang-format off -//static TdThreadOnce init = PTHREAD_ONCE_INIT; -//static void * ids = NULL; -//static TdThreadMutex mtx; -// -//void traceInit() { -// ids = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); -// taosThreadMutexInit(&mtx, NULL); -//} -//void traceCreateEnv() { -// taosThreadOnce(&init, traceInit); -//} -//void traceDestroyEnv() { -// taosThreadMutexDestroy(&mtx); -// taosHashCleanup(ids); -//} -// -//STraceId traceInitId(STraceSubId *h, STraceSubId *l) { -// STraceId id = *h; -// id = ((id << 32) & 0xFFFFFFFF) | ((*l) & 0xFFFFFFFF); -// return id; -//} -//void traceId2Str(STraceId *id, char *buf) { -// int32_t f = (*id >> 32) & 0xFFFFFFFF; -// int32_t s = (*id) & 0xFFFFFFFF; -// sprintf(buf, "%d:%d", f, s); -//} -// -//void traceSetSubId(STraceId *id, STraceSubId *subId) { -// int32_t parent = ((*id >> 32) & 0xFFFFFFFF); -// taosThreadMutexLock(&mtx); -// taosHashPut(ids, subId, sizeof(*subId), &parent, sizeof(parent)); -// taosThreadMutexUnlock(&mtx); -//} -// -//STraceSubId traceGetParentId(STraceId *id) { -// int32_t parent = ((*id >> 32) & 0xFFFFFFFF); -// taosThreadMutexLock(&mtx); -// STraceSubId *p = taosHashGet(ids, (void *)&parent, sizeof(parent)); -// parent = *p; -// taosThreadMutexUnlock(&mtx); -// -// return parent; -//} -// -//STraceSubId traceGenSubId() { -// return tGenIdPI32(); -//} -//void traceSetRootId(STraceId *traceid, int64_t rootId) { -// traceId->rootId = rootId; -//} -//int64_t traceGetRootId(STraceId *traceId); -// -//void traceSetMsgId(STraceId *traceid, int64_t msgId); -//int64_t traceGetMsgId(STraceId *traceid); -// -//char *trace2Str(STraceId *id); -// -//void traceSetSubId(STraceId *id, int32_t *subId); -// clang-format on From cc31a7c3c35e986675a34d9141b4fe87792bfba5 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 8 Jul 2022 15:38:40 +0800 Subject: [PATCH 37/38] refactor(sync): add syncRestoreFromSnapshot --- source/libs/sync/test/syncRestoreFromSnapshot.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/sync/test/syncRestoreFromSnapshot.cpp b/source/libs/sync/test/syncRestoreFromSnapshot.cpp index 97f3aa8220..470dd678b0 100644 --- a/source/libs/sync/test/syncRestoreFromSnapshot.cpp +++ b/source/libs/sync/test/syncRestoreFromSnapshot.cpp @@ -49,7 +49,7 @@ SSyncNode* createSyncNode(SWal* pWal) { void usage(char* exe) { printf("usage: %s path vgId snapshotIndex \n", exe); } int main(int argc, char** argv) { - if (argc != 3) { + if (argc != 4) { usage(argv[0]); exit(-1); } From 23fc0f0008361cc6a0e71cc9524fbfecd8e9f2d0 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 8 Jul 2022 15:54:11 +0800 Subject: [PATCH 38/38] tsdbCache/last: save ts info of each column --- source/dnode/vnode/src/inc/tsdb.h | 2 + source/dnode/vnode/src/tsdb/tsdbCache.c | 121 ++++++++++++++++---- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 7 ++ 3 files changed, 106 insertions(+), 24 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 0f65a536e0..cd2dfd3351 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -245,6 +245,8 @@ int32_t tsdbCacheRelease(SLRUCache *pCache, LRUHandle *h); int32_t tsdbCacheDelete(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey); +int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pSchema); + // structs ======================= typedef struct { int minFid; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 2086d94099..ea1dfc057f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -59,6 +59,8 @@ static void getTableCacheKey(tb_uid_t uid, int cacheType, char *key, int *len) { static void deleteTableCacheLastrow(const void *key, size_t keyLen, void *value) { taosMemoryFree(value); } +static void deleteTableCacheLast(const void *key, size_t keyLen, void *value) { taosArrayDestroy(value); } + static int32_t tsdbCacheDeleteLastrow(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey) { int32_t code = 0; char key[32] = {0}; @@ -761,7 +763,6 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, STSRow **ppRo for (int i = 0; i < nMax; ++i) { TSDBKEY maxKey = TSDBROW_KEY(max[i]); - // bool deleted = false; bool deleted = tsdbKeyDeleted(&maxKey, pSkyline, &iSkyline); if (!deleted) { // iMerge[nMerge] = i; @@ -818,12 +819,22 @@ _err: return code; } -static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, STSRow **ppRow) { - int32_t code = 0; +typedef struct { + TSKEY ts; + SColVal colVal; +} SLastCol; + +// static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, STSRow **ppRow) { +static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray) { + int32_t code = 0; + SArray *pSkyline = NULL; + STSRow *pRow = NULL; + STSRow **ppRow = &pRow; STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1); int16_t nCol = pTSchema->numOfCols; - SArray *pColArray = taosArrayInit(nCol, sizeof(SColVal)); + // SArray *pColArray = taosArrayInit(nCol, sizeof(SColVal)); + SArray *pColArray = taosArrayInit(nCol, sizeof(SLastCol)); tb_uid_t suid = getTableSuidByUid(uid, pTsdb); @@ -837,9 +848,9 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, STSRow **ppRow) { tsdbGetTbDataFromMemTable(pTsdb->imem, suid, uid, &pIMem); } - *ppRow = NULL; + *ppLastArray = NULL; - SArray *pSkyline = taosArrayInit(32, sizeof(TSDBKEY)); + pSkyline = taosArrayInit(32, sizeof(TSDBKEY)); SDelIdx delIdx; @@ -943,7 +954,6 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, STSRow **ppRow) { for (int i = 0; i < nMax; ++i) { TSDBKEY maxKey = TSDBROW_KEY(max[i]); - // bool deleted = false; bool deleted = tsdbKeyDeleted(&maxKey, pSkyline, &iSkyline); if (!deleted) { iMerge[nMerge] = iMax[i]; @@ -970,8 +980,9 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, STSRow **ppRow) { tRowMergerClear(&merger); } } else { - *ppRow = NULL; - return code; + /* *ppRow = NULL; */ + /* return code; */ + continue; } if (iCol == 0) { @@ -980,7 +991,8 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, STSRow **ppRow) { *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = maxKey}); - if (taosArrayPush(pColArray, pColVal) == NULL) { + // if (taosArrayPush(pColArray, pColVal) == NULL) { + if (taosArrayPush(pColArray, &(SLastCol){.ts = TSKEY_MAX, .colVal = *pColVal}) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } @@ -991,7 +1003,8 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, STSRow **ppRow) { for (int16_t i = iCol; i < nCol; ++i) { // tsdbRowGetColVal(*ppRow, pTSchema, i, pColVal); tTSRowGetVal(*ppRow, pTSchema, i, pColVal); - if (taosArrayPush(pColArray, pColVal) == NULL) { + // if (taosArrayPush(pColArray, pColVal) == NULL) { + if (taosArrayPush(pColArray, &(SLastCol){.ts = maxKey, .colVal = *pColVal}) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } @@ -1012,11 +1025,11 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, STSRow **ppRow) { --nilColCount; } } - /* + if (*ppRow) { taosMemoryFreeClear(*ppRow); } - */ + continue; } @@ -1024,12 +1037,16 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, STSRow **ppRow) { for (int16_t i = iCol; i < nCol; ++i) { SColVal colVal = {0}; tTSRowGetVal(*ppRow, pTSchema, i, &colVal); + TSKEY rowTs = (*ppRow)->ts; - SColVal *tColVal = (SColVal *)taosArrayGet(pColArray, i); + // SColVal *tColVal = (SColVal *)taosArrayGet(pColArray, i); + SLastCol *tTsVal = (SLastCol *)taosArrayGet(pColArray, i); + SColVal *tColVal = &tTsVal->colVal; if (!colVal.isNone && !colVal.isNull) { if (tColVal->isNull || tColVal->isNone) { - taosArraySet(pColArray, i, &colVal); + // taosArraySet(pColArray, i, &colVal); + taosArraySet(pColArray, i, &(SLastCol){.ts = rowTs, .colVal = colVal}); --nilColCount; } } else { @@ -1054,16 +1071,45 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, STSRow **ppRow) { } while (nilColCount > 0); // if () new ts row from pColArray if non empty - if (taosArrayGetSize(pColArray) == nCol) { - code = tdSTSRowNew(pColArray, pTSchema, ppRow); - if (code) goto _err; + /* if (taosArrayGetSize(pColArray) == nCol) { */ + /* code = tdSTSRowNew(pColArray, pTSchema, ppRow); */ + /* if (code) goto _err; */ + /* } */ + /* taosArrayDestroy(pColArray); */ + if (taosArrayGetSize(pColArray) <= 0) { + *ppLastArray = NULL; + taosArrayDestroy(pColArray); + } else { + *ppLastArray = pColArray; + } + if (*ppRow) { + taosMemoryFreeClear(*ppRow); + } + + for (int i = 0; i < 3; ++i) { + if (input[i].nextRowClearFn) { + input[i].nextRowClearFn(input[i].iter); + } + } + if (pSkyline) { + taosArrayDestroy(pSkyline); } - taosArrayDestroy(pColArray); taosMemoryFreeClear(pTSchema); return code; _err: taosArrayDestroy(pColArray); + if (*ppRow) { + taosMemoryFreeClear(*ppRow); + } + for (int i = 0; i < 3; ++i) { + if (input[i].nextRowClearFn) { + input[i].nextRowClearFn(input[i].iter); + } + } + if (pSkyline) { + taosArrayDestroy(pSkyline); + } taosMemoryFreeClear(pTSchema); tsdbError("vgId:%d merge last_row failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); return code; @@ -1103,6 +1149,30 @@ int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUH return code; } +int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pTSchema) { + int32_t code = 0; + int16_t nCol = taosArrayGetSize(pLastArray); + SArray *pColArray = taosArrayInit(nCol, sizeof(SColVal)); + + for (int16_t iCol = 0; iCol < nCol; ++iCol) { + SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLastArray, iCol); + SColVal *tColVal = &tTsVal->colVal; + taosArrayPush(pColArray, tColVal); + } + + code = tdSTSRowNew(pColArray, pTSchema, ppRow); + if (code) goto _err; + + taosArrayDestroy(pColArray); + + return code; + +_err: + taosArrayDestroy(pColArray); + + return code; +} + int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUHandle **handle) { int32_t code = 0; char key[32] = {0}; @@ -1115,17 +1185,20 @@ int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUHand //*ppRow = (STSRow *)taosLRUCacheValue(pCache, h); } else { - STSRow *pRow = NULL; - code = mergeLast(uid, pTsdb, &pRow); + // STSRow *pRow = NULL; + // code = mergeLast(uid, pTsdb, &pRow); + SArray *pLastArray = NULL; + code = mergeLast(uid, pTsdb, &pLastArray); // if table's empty or error, return code of -1 - if (code < 0 || pRow == NULL) { + // if (code < 0 || pRow == NULL) { + if (code < 0 || pLastArray == NULL) { *handle = NULL; return 0; } - _taos_lru_deleter_t deleter = deleteTableCacheLastrow; + _taos_lru_deleter_t deleter = deleteTableCacheLast; LRUStatus status = - taosLRUCacheInsert(pCache, key, keyLen, pRow, TD_ROW_LEN(pRow), deleter, NULL, TAOS_LRU_PRIORITY_LOW); + taosLRUCacheInsert(pCache, key, keyLen, pLastArray, pLastArray->capacity, deleter, NULL, TAOS_LRU_PRIORITY_LOW); if (status != TAOS_LRU_STATUS_OK) { code = -1; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index 30bc603b30..150ed620bf 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -128,6 +128,8 @@ int32_t tsdbRetrieveLastRow(void* pReader, SSDataBlock* pResBlock, const int32_t } pRow = (STSRow*)taosLRUCacheValue(lruCache, h); + // SArray* pLast = (SArray*)taosLRUCacheValue(lruCache, h); + // tsdbCacheLastArray2Row(pLast, &pRow, pr->pSchema); if (pRow->ts > lastKey) { // Set result row into the same rowIndex repeatly, so we need to check if the internal result row has already // appended or not. @@ -140,6 +142,7 @@ int32_t tsdbRetrieveLastRow(void* pReader, SSDataBlock* pResBlock, const int32_t lastKey = pRow->ts; } + // taosMemoryFree(pRow); tsdbCacheRelease(lruCache, h); } } else if (pr->type == LASTROW_RETRIEVE_TYPE_ALL) { @@ -158,8 +161,12 @@ int32_t tsdbRetrieveLastRow(void* pReader, SSDataBlock* pResBlock, const int32_t } pRow = (STSRow*)taosLRUCacheValue(lruCache, h); + // SArray* pLast = (SArray*)taosLRUCacheValue(lruCache, h); + // tsdbCacheLastArray2Row(pLast, &pRow, pr->pSchema); + saveOneRow(pRow, pResBlock, pr, slotIds); + // taosMemoryFree(pRow); tsdbCacheRelease(lruCache, h); pr->tableIndex += 1;